From 93ce96d01189b8f10fff212b73bcc823b2557cd1 Mon Sep 17 00:00:00 2001 From: Nicola Murino Date: Wed, 29 Jul 2020 21:56:56 +0200 Subject: [PATCH] add support for the venerable FTP protocol Fixes #46 --- .github/workflows/development.yml | 3 +- README.md | 8 +- cmd/portable.go | 38 +- common/actions_test.go | 2 +- common/common.go | 81 +- common/common_test.go | 63 +- common/tlsutils.go | 2 +- common/tlsutils_test.go | 4 +- config/config.go | 38 +- config/config_test.go | 41 +- dataprovider/user.go | 3 +- docs/full-configuration.md | 10 + docs/portable-mode.md | 22 +- ftpd/ftpd.go | 82 ++ ftpd/ftpd_test.go | 1263 +++++++++++++++++++++++++++++ ftpd/handler.go | 387 +++++++++ ftpd/internal_test.go | 441 ++++++++++ ftpd/server.go | 190 +++++ ftpd/transfer.go | 128 +++ go.mod | 10 +- go.sum | 152 +++- httpd/httpd_test.go | 10 +- httpd/internal_test.go | 14 +- metrics/metrics.go | 37 +- metrics/metrics_disabled.go | 4 + service/service.go | 32 +- service/service_portable.go | 73 +- service/service_windows.go | 5 + service/sighup_unix.go | 5 + sftpd/handler.go | 14 +- sftpd/internal_test.go | 28 +- sftpd/scp.go | 6 +- sftpd/server.go | 5 +- sftpd/sftpd_test.go | 8 +- sftpd/ssh_cmd.go | 8 +- sftpd/transfer.go | 2 +- sftpgo.json | 14 + vfs/osfs.go | 2 +- 38 files changed, 3075 insertions(+), 160 deletions(-) create mode 100644 ftpd/ftpd.go create mode 100644 ftpd/ftpd_test.go create mode 100644 ftpd/handler.go create mode 100644 ftpd/internal_test.go create mode 100644 ftpd/server.go create mode 100644 ftpd/transfer.go diff --git a/.github/workflows/development.yml b/.github/workflows/development.yml index cf91790f..372c7b91 100644 --- a/.github/workflows/development.yml +++ b/.github/workflows/development.yml @@ -61,6 +61,7 @@ jobs: go test -v -timeout 1m ./common -covermode=atomic go test -v -timeout 5m ./httpd -covermode=atomic go test -v -timeout 5m ./sftpd -covermode=atomic + go test -v -timeout 5m ./ftpd -covermode=atomic env: SFTPGO_DATA_PROVIDER__DRIVER: bolt SFTPGO_DATA_PROVIDER__NAME: 'sftpgo_bolt.db' @@ -177,4 +178,4 @@ jobs: - name: Run golangci-lint uses: golangci/golangci-lint-action@v1 with: - version: v1.27 + version: v1.29 diff --git a/README.md b/README.md index 6202b38c..15857f03 100644 --- a/README.md +++ b/README.md @@ -33,9 +33,10 @@ Fully featured and highly configurable SFTP server, written in Go - Atomic uploads are configurable. - Support for Git repositories over SSH. - SCP and rsync are supported. -- Support for serving local filesystem, S3 Compatible Object Storage and Google Cloud Storage over SFTP/SCP. +- FTP/S is supported. +- Support for serving local filesystem, S3 Compatible Object Storage and Google Cloud Storage over SFTP/SCP/FTP. - [Prometheus metrics](./docs/metrics.md) are exposed. -- Support for HAProxy PROXY protocol: you can proxy and/or load balance the SFTP/SCP service without losing the information about the client's address. +- Support for HAProxy PROXY protocol: you can proxy and/or load balance the SFTP/SCP/FTP service without losing the information about the client's address. - [REST API](./docs/rest-api.md) for users and folders management, backup, restore and real time reports of the active connections with possibility of forcibly closing a connection. - [Web based administration interface](./docs/web-admin.md) to easily manage users, folders and connections. - Easy [migration](./examples/rest-api-cli#convert-users-from-other-stores) from Linux system user accounts. @@ -51,7 +52,8 @@ SFTPGo is developed and tested on Linux. After each commit, the code is automati ## Requirements - Go 1.13 or higher as build only dependency. -- A suitable SQL server or key/value store to use as data provider: PostgreSQL 9.4+ or MySQL 5.6+ or SQLite 3.x or bbolt 1.3.x +- A suitable SQL server to use as data provider: PostgreSQL 9.4+ or MySQL 5.6+ or SQLite 3.x. +- The SQL server is optional: you can choose to use an embedded bolt database as key/value store or an in memory data provider. ## Installation diff --git a/cmd/portable.go b/cmd/portable.go index 4cd4d9cf..6edb0a56 100644 --- a/cmd/portable.go +++ b/cmd/portable.go @@ -13,6 +13,7 @@ import ( "github.com/spf13/cobra" + "github.com/drakkan/sftpgo/common" "github.com/drakkan/sftpgo/dataprovider" "github.com/drakkan/sftpgo/service" "github.com/drakkan/sftpgo/sftpd" @@ -49,6 +50,9 @@ var ( portableGCSAutoCredentials int portableGCSStorageClass string portableGCSKeyPrefix string + portableFTPDPort int + portableFTPSCert string + portableFTPSKey string portableCmd = &cobra.Command{ Use: "portable", Short: "Serve a single directory", @@ -88,6 +92,14 @@ Please take a look at the usage below to customize the serving parameters`, portableGCSCredentials = base64.StdEncoding.EncodeToString(creds) portableGCSAutoCredentials = 0 } + if portableFTPDPort >= 0 && len(portableFTPSCert) > 0 && len(portableFTPSKey) > 0 { + _, err := common.NewCertManager(portableFTPSCert, portableFTPSKey, "FTP portable") + if err != nil { + fmt.Printf("Unable to load FTPS key pair, cert file %#v key file %#v error: %v\n", + portableFTPSCert, portableFTPSKey, err) + os.Exit(1) + } + } service := service.Service{ ConfigDir: filepath.Clean(defaultConfigDir), ConfigFile: defaultConfigName, @@ -133,10 +145,12 @@ Please take a look at the usage below to customize the serving parameters`, }, }, } - if err := service.StartPortableMode(portableSFTPDPort, portableSSHCommands, portableAdvertiseService, - portableAdvertiseCredentials); err == nil { + if err := service.StartPortableMode(portableSFTPDPort, portableFTPDPort, portableSSHCommands, portableAdvertiseService, + portableAdvertiseCredentials, portableFTPSCert, portableFTPSKey); err == nil { service.Wait() - os.Exit(0) + if service.Error == nil { + os.Exit(0) + } } os.Exit(1) }, @@ -150,7 +164,9 @@ func init() { This can be an absolute path or a path relative to the current directory `) - portableCmd.Flags().IntVarP(&portableSFTPDPort, "sftpd-port", "s", 0, "0 means a random unprivileged port") + portableCmd.Flags().IntVarP(&portableSFTPDPort, "sftpd-port", "s", 0, "0 means a random unprivileged port") + portableCmd.Flags().IntVar(&portableFTPDPort, "ftpd-port", -1, `0 means a random unprivileged port, +< 0 disabled`) portableCmd.Flags().StringSliceVarP(&portableSSHCommands, "ssh-commands", "c", sftpd.GetDefaultSSHCommands(), `SSH commands to enable. "*" means any supported SSH command @@ -177,13 +193,13 @@ insensitive. The format is /dir::ext1,ext2. For example: "/somedir::.jpg,.png"`) portableCmd.Flags().BoolVarP(&portableAdvertiseService, "advertise-service", "S", false, - `Advertise SFTP service using multicast -DNS`) + `Advertise SFTP/FTP service using +multicast DNS`) portableCmd.Flags().BoolVarP(&portableAdvertiseCredentials, "advertise-credentials", "C", false, - `If the SFTP service is advertised via -multicast DNS, this flag allows to put -username/password inside the advertised -TXT record`) + `If the SFTP/FTP service is +advertised via multicast DNS, this +flag allows to put username/password +inside the advertised TXT record`) portableCmd.Flags().IntVarP(&portableFsProvider, "fs-provider", "f", 0, `0 means local filesystem, 1 Amazon S3 compatible, 2 Google Cloud Storage`) @@ -210,6 +226,8 @@ file`) portableCmd.Flags().IntVar(&portableGCSAutoCredentials, "gcs-automatic-credentials", 1, `0 means explicit credentials using a JSON credentials file, 1 automatic `) + portableCmd.Flags().StringVar(&portableFTPSCert, "ftpd-cert", "", "Path to the certificate file for FTPS") + portableCmd.Flags().StringVar(&portableFTPSKey, "ftpd-key", "", "Path to the key file for FTPS") rootCmd.AddCommand(portableCmd) } diff --git a/common/actions_test.go b/common/actions_test.go index 475fffda..5a9228d9 100644 --- a/common/actions_test.go +++ b/common/actions_test.go @@ -167,7 +167,7 @@ func TestPreDeleteAction(t *testing.T) { c := NewBaseConnection("id", ProtocolSFTP, user, fs) testfile := filepath.Join(user.HomeDir, "testfile") - err = ioutil.WriteFile(testfile, []byte("test"), 0666) + err = ioutil.WriteFile(testfile, []byte("test"), os.ModePerm) assert.NoError(t, err) info, err := os.Stat(testfile) assert.NoError(t, err) diff --git a/common/common.go b/common/common.go index 602c3fc0..4fc4e758 100644 --- a/common/common.go +++ b/common/common.go @@ -18,6 +18,7 @@ import ( // constants const ( + logSender = "common" uploadLogSender = "Upload" downloadLogSender = "Download" renameLogSender = "Rename" @@ -35,7 +36,7 @@ const ( operationRename = "rename" operationSSHCmd = "ssh_cmd" chtimesFormat = "2006-01-02T15:04:05" // YYYY-MM-DDTHH:MM:SS - idleTimeoutCheckInterval = 5 * time.Minute + idleTimeoutCheckInterval = 3 * time.Minute ) // Stat flags @@ -56,6 +57,7 @@ const ( ProtocolSFTP = "SFTP" ProtocolSCP = "SCP" ProtocolSSH = "SSH" + ProtocolFTP = "FTP" ) // Upload modes @@ -84,12 +86,13 @@ var ( QuotaScans ActiveScans idleTimeoutTicker *time.Ticker idleTimeoutTickerDone chan bool - supportedProcols = []string{ProtocolSFTP, ProtocolSCP, ProtocolSSH} + supportedProcols = []string{ProtocolSFTP, ProtocolSCP, ProtocolSSH, ProtocolFTP} ) // Initialize sets the common configuration func Initialize(c Configuration) { Config = c + Config.idleLoginTimeout = 2 * time.Minute Config.idleTimeoutAsDuration = time.Duration(Config.IdleTimeout) * time.Minute if Config.IdleTimeout > 0 { startIdleTimeoutTicker(idleTimeoutCheckInterval) @@ -220,6 +223,7 @@ type Configuration struct { // connection will be rejected. ProxyAllowed []string `json:"proxy_allowed" mapstructure:"proxy_allowed"` idleTimeoutAsDuration time.Duration + idleLoginTimeout time.Duration } // IsAtomicUploadEnabled returns true if atomic upload is enabled @@ -291,15 +295,35 @@ func (conns *ActiveConnections) Add(c ActiveConnection) { logger.Debug(c.GetProtocol(), c.GetID(), "connection added, num open connections: %v", len(conns.connections)) } -// Remove removes a connection from the active ones -func (conns *ActiveConnections) Remove(c ActiveConnection) { +// Swap replaces an existing connection with the given one. +// This method is useful if you have to change some connection details +// for example for FTP is used to update the connection once the user +// authenticates +func (conns *ActiveConnections) Swap(c ActiveConnection) error { conns.Lock() defer conns.Unlock() + for idx, conn := range conns.connections { + if conn.GetID() == c.GetID() { + conn = nil + conns.connections[idx] = c + return nil + } + } + return errors.New("connection to swap not found") +} + +// Remove removes a connection from the active ones +func (conns *ActiveConnections) Remove(connectionID string) { + conns.Lock() + defer conns.Unlock() + + var c ActiveConnection indexToRemove := -1 - for i, v := range conns.connections { - if v.GetID() == c.GetID() { + for i, conn := range conns.connections { + if conn.GetID() == connectionID { indexToRemove = i + c = conn break } } @@ -307,19 +331,20 @@ func (conns *ActiveConnections) Remove(c ActiveConnection) { conns.connections[indexToRemove] = conns.connections[len(conns.connections)-1] conns.connections[len(conns.connections)-1] = nil conns.connections = conns.connections[:len(conns.connections)-1] + metrics.UpdateActiveConnectionsSize(len(conns.connections)) logger.Debug(c.GetProtocol(), c.GetID(), "connection removed, num open connections: %v", len(conns.connections)) + // we have finished to send data here and most of the time the underlying network connection + // is already closed. Sometime a client can still be reading the last sended data, so we set + // a deadline instead of directly closing the network connection. + // Setting a deadline on an already closed connection has no effect. + // We only need to ensure that a connection will not remain indefinitely open and so the + // underlying file descriptor is not released. + // This should protect us against buggy clients and edge cases. + c.SetConnDeadline() } else { - logger.Warn(c.GetProtocol(), c.GetID(), "connection to remove not found!") + logger.Warn(logSender, "", "connection to remove with id %#v not found!", connectionID) } - // we have finished to send data here and most of the time the underlying network connection - // is already closed. Sometime a client can still be reading the last sended data, so we set - // a deadline instead of directly closing the network connection. - // Setting a deadline on an already closed connection has no effect. - // We only need to ensure that a connection will not remain indefinitely open and so the - // underlying file descriptor is not released. - // This should protect us against buggy clients and edge cases. - c.SetConnDeadline() } // Close closes an active connection. @@ -330,10 +355,10 @@ func (conns *ActiveConnections) Close(connectionID string) bool { for _, c := range conns.connections { if c.GetID() == connectionID { - defer func() { - err := c.Disconnect() - logger.Debug(c.GetProtocol(), c.GetID(), "close connection requested, close err: %v", err) - }() + defer func(conn ActiveConnection) { + err := conn.Disconnect() + logger.Debug(conn.GetProtocol(), conn.GetID(), "close connection requested, close err: %v", err) + }(c) result = true break } @@ -348,11 +373,19 @@ func (conns *ActiveConnections) checkIdleConnections() { for _, c := range conns.connections { idleTime := time.Since(c.GetLastActivity()) - if idleTime > Config.idleTimeoutAsDuration { - defer func() { - err := c.Disconnect() - logger.Debug(c.GetProtocol(), c.GetID(), "close idle connection, idle time: %v, close err: %v", idleTime, err) - }() + isUnauthenticatedFTPUser := (c.GetProtocol() == ProtocolFTP && len(c.GetUsername()) == 0) + + if idleTime > Config.idleTimeoutAsDuration || (isUnauthenticatedFTPUser && idleTime > Config.idleLoginTimeout) { + defer func(conn ActiveConnection, isFTPNoAuth bool) { + err := conn.Disconnect() + logger.Debug(conn.GetProtocol(), conn.GetID(), "close idle connection, idle time: %v, username: %#v close err: %v", + idleTime, conn.GetUsername(), err) + if isFTPNoAuth { + logger.ConnectionFailedLog("", utils.GetIPFromRemoteAddress(c.GetRemoteAddress()), + "no_auth_tryed", "client idle") + metrics.AddNoAuthTryed() + } + }(c, isUnauthenticatedFTPUser) } } diff --git a/common/common_test.go b/common/common_test.go index bbbbd647..84c47652 100644 --- a/common/common_test.go +++ b/common/common_test.go @@ -19,7 +19,7 @@ import ( ) const ( - logSender = "common_test" + logSenderTest = "common_test" httpAddr = "127.0.0.1:9999" httpProxyAddr = "127.0.0.1:7777" configDir = ".." @@ -37,8 +37,18 @@ type fakeConnection struct { sshCommand string } +func (c *fakeConnection) AddUser(user dataprovider.User) error { + fs, err := user.GetFilesystem(c.GetID()) + if err != nil { + return err + } + c.BaseConnection.User = user + c.BaseConnection.Fs = fs + return nil +} + func (c *fakeConnection) Disconnect() error { - Connections.Remove(c) + Connections.Remove(c.GetID()) return nil } @@ -166,16 +176,30 @@ func TestIdleConnections(t *testing.T) { user := dataprovider.User{ Username: username, } - c := NewBaseConnection("id", ProtocolSFTP, user, nil) + c := NewBaseConnection("id1", ProtocolSFTP, user, nil) c.lastActivity = time.Now().Add(-24 * time.Hour).UnixNano() fakeConn := &fakeConnection{ BaseConnection: c, } Connections.Add(fakeConn) assert.Equal(t, Connections.GetActiveSessions(username), 1) + c = NewBaseConnection("id2", ProtocolFTP, dataprovider.User{}, nil) + c.lastActivity = time.Now().UnixNano() + fakeConn = &fakeConnection{ + BaseConnection: c, + } + Connections.Add(fakeConn) + assert.Equal(t, Connections.GetActiveSessions(username), 1) + assert.Len(t, Connections.GetStats(), 2) + startIdleTimeoutTicker(100 * time.Millisecond) assert.Eventually(t, func() bool { return Connections.GetActiveSessions(username) == 0 }, 1*time.Second, 200*time.Millisecond) stopIdleTimeoutTicker() + assert.Len(t, Connections.GetStats(), 1) + c.lastActivity = time.Now().Add(-24 * time.Hour).UnixNano() + startIdleTimeoutTicker(100 * time.Millisecond) + assert.Eventually(t, func() bool { return len(Connections.GetStats()) == 0 }, 1*time.Second, 200*time.Millisecond) + stopIdleTimeoutTicker() Config = configCopy } @@ -192,7 +216,34 @@ func TestCloseConnection(t *testing.T) { assert.Eventually(t, func() bool { return len(Connections.GetStats()) == 0 }, 300*time.Millisecond, 50*time.Millisecond) res = Connections.Close(fakeConn.GetID()) assert.False(t, res) - Connections.Remove(fakeConn) + Connections.Remove(fakeConn.GetID()) +} + +func TestSwapConnection(t *testing.T) { + c := NewBaseConnection("id", ProtocolFTP, dataprovider.User{}, nil) + fakeConn := &fakeConnection{ + BaseConnection: c, + } + Connections.Add(fakeConn) + if assert.Len(t, Connections.GetStats(), 1) { + assert.Equal(t, "", Connections.GetStats()[0].Username) + } + c = NewBaseConnection("id", ProtocolFTP, dataprovider.User{ + Username: userTestUsername, + }, nil) + fakeConn = &fakeConnection{ + BaseConnection: c, + } + err := Connections.Swap(fakeConn) + assert.NoError(t, err) + if assert.Len(t, Connections.GetStats(), 1) { + assert.Equal(t, userTestUsername, Connections.GetStats()[0].Username) + } + res := Connections.Close(fakeConn.GetID()) + assert.True(t, res) + assert.Eventually(t, func() bool { return len(Connections.GetStats()) == 0 }, 300*time.Millisecond, 50*time.Millisecond) + err = Connections.Swap(fakeConn) + assert.Error(t, err) } func TestAtomicUpload(t *testing.T) { @@ -255,8 +306,8 @@ func TestConnectionStatus(t *testing.T) { err = t2.Close() assert.NoError(t, err) - Connections.Remove(fakeConn1) - Connections.Remove(fakeConn2) + Connections.Remove(fakeConn1.GetID()) + Connections.Remove(fakeConn2.GetID()) stats = Connections.GetStats() assert.Len(t, stats, 0) } diff --git a/common/tlsutils.go b/common/tlsutils.go index e7f8651e..d2f029a2 100644 --- a/common/tlsutils.go +++ b/common/tlsutils.go @@ -19,7 +19,7 @@ type CertManager struct { func (m *CertManager) LoadCertificate(logSender string) error { newCert, err := tls.LoadX509KeyPair(m.certPath, m.keyPath) if err != nil { - logger.Warn(logSender, "", "unable to load X509 ket pair, cert file %#v key file %#v error: %v", + logger.Warn(logSender, "", "unable to load X509 key pair, cert file %#v key file %#v error: %v", m.certPath, m.keyPath, err) return err } diff --git a/common/tlsutils_test.go b/common/tlsutils_test.go index ad11874d..b0efb848 100644 --- a/common/tlsutils_test.go +++ b/common/tlsutils_test.go @@ -43,7 +43,7 @@ func TestLoadCertificate(t *testing.T) { assert.NoError(t, err) err = ioutil.WriteFile(keyPath, []byte(httpsKey), os.ModePerm) assert.NoError(t, err) - certManager, err := NewCertManager(certPath, keyPath, logSender) + certManager, err := NewCertManager(certPath, keyPath, logSenderTest) assert.NoError(t, err) certFunc := certManager.GetCertificateFunc() if assert.NotNil(t, certFunc) { @@ -63,7 +63,7 @@ func TestLoadCertificate(t *testing.T) { } func TestLoadInvalidCert(t *testing.T) { - certManager, err := NewCertManager("test.crt", "test.key", logSender) + certManager, err := NewCertManager("test.crt", "test.key", logSenderTest) assert.Error(t, err) assert.Nil(t, certManager) } diff --git a/config/config.go b/config/config.go index 2f799828..d5b96d7f 100644 --- a/config/config.go +++ b/config/config.go @@ -9,6 +9,7 @@ import ( "github.com/drakkan/sftpgo/common" "github.com/drakkan/sftpgo/dataprovider" + "github.com/drakkan/sftpgo/ftpd" "github.com/drakkan/sftpgo/httpclient" "github.com/drakkan/sftpgo/httpd" "github.com/drakkan/sftpgo/logger" @@ -28,13 +29,15 @@ const ( ) var ( - globalConf globalConfig - defaultBanner = fmt.Sprintf("SFTPGo_%v", version.Get().Version) + globalConf globalConfig + defaultSFTPDBanner = fmt.Sprintf("SFTPGo_%v", version.Get().Version) + defaultFTPDBanner = fmt.Sprintf("SFTPGo %v ready", version.Get().Version) ) type globalConfig struct { Common common.Configuration `json:"common" mapstructure:"common"` SFTPD sftpd.Configuration `json:"sftpd" mapstructure:"sftpd"` + FTPD ftpd.Configuration `json:"ftpd" mapstructure:"ftpd"` ProviderConf dataprovider.Config `json:"data_provider" mapstructure:"data_provider"` HTTPDConfig httpd.Conf `json:"httpd" mapstructure:"httpd"` HTTPConfig httpclient.Config `json:"http" mapstructure:"http"` @@ -55,7 +58,7 @@ func init() { ProxyAllowed: []string{}, }, SFTPD: sftpd.Configuration{ - Banner: defaultBanner, + Banner: defaultSFTPDBanner, BindPort: 2022, BindAddress: "", MaxAuthTries: 0, @@ -68,6 +71,20 @@ func init() { EnabledSSHCommands: sftpd.GetDefaultSSHCommands(), KeyboardInteractiveHook: "", }, + FTPD: ftpd.Configuration{ + BindPort: 0, + BindAddress: "", + Banner: defaultFTPDBanner, + BannerFile: "", + ActiveTransfersPortNon20: false, + ForcePassiveIP: "", + PassivePortRange: ftpd.PortRange{ + Start: 50000, + End: 50100, + }, + CertificateFile: "", + CertificateKeyFile: "", + }, ProviderConf: dataprovider.Config{ Driver: "sqlite", Name: "sftpgo.db", @@ -136,6 +153,16 @@ func SetSFTPDConfig(config sftpd.Configuration) { globalConf.SFTPD = config } +// GetFTPDConfig returns the configuration for the FTP server +func GetFTPDConfig() ftpd.Configuration { + return globalConf.FTPD +} + +// SetFTPDConfig sets the configuration for the FTP server +func SetFTPDConfig(config ftpd.Configuration) { + globalConf.FTPD = config +} + // GetHTTPDConfig returns the configuration for the HTTP server func GetHTTPDConfig() httpd.Conf { return globalConf.HTTPDConfig @@ -193,7 +220,10 @@ func LoadConfig(configDir, configName string) error { } checkCommonParamsCompatibility() if strings.TrimSpace(globalConf.SFTPD.Banner) == "" { - globalConf.SFTPD.Banner = defaultBanner + globalConf.SFTPD.Banner = defaultSFTPDBanner + } + if strings.TrimSpace(globalConf.FTPD.Banner) == "" { + globalConf.FTPD.Banner = defaultFTPDBanner } if len(globalConf.ProviderConf.UsersBaseDir) > 0 && !utils.IsFileInputValid(globalConf.ProviderConf.UsersBaseDir) { err = fmt.Errorf("invalid users base dir %#v will be ignored", globalConf.ProviderConf.UsersBaseDir) diff --git a/config/config_test.go b/config/config_test.go index 13a357cf..f68d7a3f 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -13,6 +13,7 @@ import ( "github.com/drakkan/sftpgo/common" "github.com/drakkan/sftpgo/config" "github.com/drakkan/sftpgo/dataprovider" + "github.com/drakkan/sftpgo/ftpd" "github.com/drakkan/sftpgo/httpclient" "github.com/drakkan/sftpgo/httpd" "github.com/drakkan/sftpgo/sftpd" @@ -36,11 +37,11 @@ func TestLoadConfigTest(t *testing.T) { configFilePath := filepath.Join(configDir, confName) err = config.LoadConfig(configDir, tempConfigName) assert.NotNil(t, err) - err = ioutil.WriteFile(configFilePath, []byte("{invalid json}"), 0666) + err = ioutil.WriteFile(configFilePath, []byte("{invalid json}"), os.ModePerm) assert.NoError(t, err) err = config.LoadConfig(configDir, tempConfigName) assert.NotNil(t, err) - err = ioutil.WriteFile(configFilePath, []byte("{\"sftpd\": {\"bind_port\": \"a\"}}"), 0666) + err = ioutil.WriteFile(configFilePath, []byte("{\"sftpd\": {\"bind_port\": \"a\"}}"), os.ModePerm) assert.NoError(t, err) err = config.LoadConfig(configDir, tempConfigName) assert.NotNil(t, err) @@ -59,7 +60,7 @@ func TestEmptyBanner(t *testing.T) { c := make(map[string]sftpd.Configuration) c["sftpd"] = sftpdConf jsonConf, _ := json.Marshal(c) - err = ioutil.WriteFile(configFilePath, jsonConf, 0666) + err = ioutil.WriteFile(configFilePath, jsonConf, os.ModePerm) assert.NoError(t, err) err = config.LoadConfig(configDir, tempConfigName) assert.NoError(t, err) @@ -67,6 +68,20 @@ func TestEmptyBanner(t *testing.T) { assert.NotEmpty(t, strings.TrimSpace(sftpdConf.Banner)) err = os.Remove(configFilePath) assert.NoError(t, err) + + ftpdConf := config.GetFTPDConfig() + ftpdConf.Banner = " " + c1 := make(map[string]ftpd.Configuration) + c1["ftpd"] = ftpdConf + jsonConf, _ = json.Marshal(c1) + err = ioutil.WriteFile(configFilePath, jsonConf, os.ModePerm) + assert.NoError(t, err) + err = config.LoadConfig(configDir, tempConfigName) + assert.NoError(t, err) + ftpdConf = config.GetFTPDConfig() + assert.NotEmpty(t, strings.TrimSpace(ftpdConf.Banner)) + err = os.Remove(configFilePath) + assert.NoError(t, err) } func TestInvalidUploadMode(t *testing.T) { @@ -81,7 +96,7 @@ func TestInvalidUploadMode(t *testing.T) { c["common"] = commonConf jsonConf, err := json.Marshal(c) assert.NoError(t, err) - err = ioutil.WriteFile(configFilePath, jsonConf, 0666) + err = ioutil.WriteFile(configFilePath, jsonConf, os.ModePerm) assert.NoError(t, err) err = config.LoadConfig(configDir, tempConfigName) assert.NotNil(t, err) @@ -101,7 +116,7 @@ func TestInvalidExternalAuthScope(t *testing.T) { c["data_provider"] = providerConf jsonConf, err := json.Marshal(c) assert.NoError(t, err) - err = ioutil.WriteFile(configFilePath, jsonConf, 0666) + err = ioutil.WriteFile(configFilePath, jsonConf, os.ModePerm) assert.NoError(t, err) err = config.LoadConfig(configDir, tempConfigName) assert.NotNil(t, err) @@ -121,7 +136,7 @@ func TestInvalidCredentialsPath(t *testing.T) { c["data_provider"] = providerConf jsonConf, err := json.Marshal(c) assert.NoError(t, err) - err = ioutil.WriteFile(configFilePath, jsonConf, 0666) + err = ioutil.WriteFile(configFilePath, jsonConf, os.ModePerm) assert.NoError(t, err) err = config.LoadConfig(configDir, tempConfigName) assert.NotNil(t, err) @@ -141,7 +156,7 @@ func TestInvalidProxyProtocol(t *testing.T) { c["common"] = commonConf jsonConf, err := json.Marshal(c) assert.NoError(t, err) - err = ioutil.WriteFile(configFilePath, jsonConf, 0666) + err = ioutil.WriteFile(configFilePath, jsonConf, os.ModePerm) assert.NoError(t, err) err = config.LoadConfig(configDir, tempConfigName) assert.NotNil(t, err) @@ -161,7 +176,7 @@ func TestInvalidUsersBaseDir(t *testing.T) { c["data_provider"] = providerConf jsonConf, err := json.Marshal(c) assert.NoError(t, err) - err = ioutil.WriteFile(configFilePath, jsonConf, 0666) + err = ioutil.WriteFile(configFilePath, jsonConf, os.ModePerm) assert.NoError(t, err) err = config.LoadConfig(configDir, tempConfigName) assert.NotNil(t, err) @@ -187,7 +202,7 @@ func TestCommonParamsCompatibility(t *testing.T) { c["sftpd"] = sftpdConf jsonConf, err := json.Marshal(c) assert.NoError(t, err) - err = ioutil.WriteFile(configFilePath, jsonConf, 0666) + err = ioutil.WriteFile(configFilePath, jsonConf, os.ModePerm) assert.NoError(t, err) err = config.LoadConfig(configDir, tempConfigName) assert.NoError(t, err) @@ -223,7 +238,7 @@ func TestHostKeyCompatibility(t *testing.T) { c["sftpd"] = sftpdConf jsonConf, err := json.Marshal(c) assert.NoError(t, err) - err = ioutil.WriteFile(configFilePath, jsonConf, 0666) + err = ioutil.WriteFile(configFilePath, jsonConf, os.ModePerm) assert.NoError(t, err) err = config.LoadConfig(configDir, tempConfigName) assert.NoError(t, err) @@ -252,4 +267,10 @@ func TestSetGetConfig(t *testing.T) { commonConf.IdleTimeout = 10 config.SetCommonConfig(commonConf) assert.Equal(t, commonConf.IdleTimeout, config.GetCommonConfig().IdleTimeout) + ftpdConf := config.GetFTPDConfig() + ftpdConf.CertificateFile = "cert" + ftpdConf.CertificateKeyFile = "key" + config.SetFTPDConfig(ftpdConf) + assert.Equal(t, ftpdConf.CertificateFile, config.GetFTPDConfig().CertificateFile) + assert.Equal(t, ftpdConf.CertificateKeyFile, config.GetFTPDConfig().CertificateKeyFile) } diff --git a/dataprovider/user.go b/dataprovider/user.go index 31eee546..c883fdf1 100644 --- a/dataprovider/user.go +++ b/dataprovider/user.go @@ -46,13 +46,14 @@ const ( PermChtimes = "chtimes" ) -// Available SSH login methods +// Available login methods const ( SSHLoginMethodPublicKey = "publickey" SSHLoginMethodPassword = "password" SSHLoginMethodKeyboardInteractive = "keyboard-interactive" SSHLoginMethodKeyAndPassword = "publickey+password" SSHLoginMethodKeyAndKeyboardInt = "publickey+keyboard-interactive" + FTPLoginMethodPassword = "ftp-password" ) var ( diff --git a/docs/full-configuration.md b/docs/full-configuration.md index 5b7df18d..32ea4d65 100644 --- a/docs/full-configuration.md +++ b/docs/full-configuration.md @@ -78,6 +78,16 @@ The configuration file contains the following sections: - `keyboard_interactive_auth_hook`, string. Absolute path to an external program or an HTTP URL to invoke for keyboard interactive authentication. See the "Keyboard Interactive Authentication" paragraph for more details. - `proxy_protocol`, integer. Deprecated, please use the same key in `common` section. - `proxy_allowed`, list of strings. Deprecated, please use the same key in `common` section. +- **"ftpd"**, the configuration for the FTP server + - `bind_port`, integer. The port used for serving FTP requests. 0 means disabled. Default: 0. + - `bind_address`, string. Leave blank to listen on all available network interfaces. Default: "". + - `banner`, string. Greeting banner displayed when a connection first comes in. Leave empty to use the default banner. Default `SFTPGo ready`, for example `SFTPGo 1.0.0-dev ready`. + - `banner_file`, path to the banner file. The contents of the specified file, if any, are diplayed when someone connects to the server. It can be a path relative to the config dir or an absolute one. If set, it overrides the banner string provided by the `banner` option. Leave empty to disable. + - `active_transfers_port_non_20`, boolean. Do not impose the port 20 for active data transfers. Enabling this option allows to run SFTPGo with less privilege. Default: false. + - `force_passive_ip`, ip address. External IP address to expose for passive connections. Leavy empty to autodetect. Defaut: "". + - `passive_port_range`, struct containing the key `start` and `end`. Port Range for data connections. Random if not specified. Default range is 50000-50100. + - `certificate_file`, string. Certificate for FTPS. This can be an absolute path or a path relative to the config dir. + - `certificate_key_file`, string. Private key matching the above certificate. This can be an absolute path or a path relative to the config dir. If both the certificate and the private key are provided, the server will accept both plain FTP an explicit FTP over TLS. Certificate and key files can be reloaded on demand sending a `SIGHUP` signal on Unix based systems and a `paramchange` request to the running service on Windows. - **"data_provider"**, the configuration for the data provider - `driver`, string. Supported drivers are `sqlite`, `mysql`, `postgresql`, `bolt`, `memory` - `name`, string. Database name. For driver `sqlite` this can be the database name relative to the config dir or the absolute path to the SQLite database. For driver `memory` this is the (optional) path relative to the config dir or the absolute path to the users dump, obtained using the `dumpdata` REST API, to load. This dump will be loaded at startup and can be reloaded on demand sending a `SIGHUP` signal on Unix based systems and a `paramchange` request to the running service on Windows. The `memory` provider will not modify the provided file so quota usage and last login will not be persisted diff --git a/docs/portable-mode.md b/docs/portable-mode.md index ba111c69..2100d474 100644 --- a/docs/portable-mode.md +++ b/docs/portable-mode.md @@ -15,12 +15,12 @@ Usage: sftpgo portable [flags] Flags: - -C, --advertise-credentials If the SFTP service is advertised via - multicast DNS, this flag allows to put - username/password inside the advertised - TXT record - -S, --advertise-service Advertise SFTP service using multicast - DNS + -C, --advertise-credentials If the SFTP/FTP service is + advertised via multicast DNS, this + flag allows to put username/password + inside the advertised TXT record + -S, --advertise-service Advertise SFTP/FTP service using + multicast DNS --allowed-extensions stringArray Allowed file extensions case insensitive. The format is /dir::ext1,ext2. @@ -36,6 +36,10 @@ Flags: -f, --fs-provider int 0 means local filesystem, 1 Amazon S3 compatible, 2 Google Cloud Storage + --ftpd-cert string Path to the certificate file for FTPS + --ftpd-key string Path to the key file for FTPS + --ftpd-port int 0 means a random unprivileged port, + < 0 disabled (default -1) --gcs-automatic-credentials int 0 means explicit credentials using a JSON credentials file, 1 automatic (default 1) @@ -67,7 +71,7 @@ Flags: parallel (default 2) --s3-upload-part-size int The buffer size for multipart uploads (MB) (default 5) - -s, --sftpd-port int 0 means a random unprivileged port + -s, --sftpd-port int 0 means a random unprivileged port -c, --ssh-commands strings SSH commands to enable. "*" means any supported SSH command including scp @@ -76,9 +80,9 @@ Flags: value ``` -In portable mode, SFTPGo can advertise the SFTP service and, optionally, the credentials via multicast DNS, so there is a standard way to discover the service and to automatically connect to it. +In portable mode, SFTPGo can advertise the SFTP/FTP services and, optionally, the credentials via multicast DNS, so there is a standard way to discover the service and to automatically connect to it. -Here is an example of the advertised service including credentials as seen using `avahi-browse`: +Here is an example of the advertised SFTP service including credentials as seen using `avahi-browse`: ```console = enp0s31f6 IPv4 SFTPGo portable 53705 SFTP File Transfer local diff --git a/ftpd/ftpd.go b/ftpd/ftpd.go new file mode 100644 index 00000000..02bf52bc --- /dev/null +++ b/ftpd/ftpd.go @@ -0,0 +1,82 @@ +// Package ftpd implements the FTP protocol +package ftpd + +import ( + "path/filepath" + + ftpserver "github.com/fclairamb/ftpserverlib" + + "github.com/drakkan/sftpgo/logger" + "github.com/drakkan/sftpgo/utils" +) + +const ( + logSender = "ftpd" +) + +var ( + server *Server +) + +// PortRange defines a port range +type PortRange struct { + // Range start + Start int `json:"start" mapstructure:"start"` + // Range end + End int `json:"end" mapstructure:"end"` +} + +// Configuration defines the configuration for the ftp server +type Configuration struct { + // The port used for serving FTP requests + BindPort int `json:"bind_port" mapstructure:"bind_port"` + // The address to listen on. A blank value means listen on all available network interfaces. + BindAddress string `json:"bind_address" mapstructure:"bind_address"` + // External IP address to expose for passive connections. + ForcePassiveIP string `json:"force_passive_ip" mapstructure:"force_passive_ip"` + // Greeting banner displayed when a connection first comes in + Banner string `json:"banner" mapstructure:"banner"` + // the contents of the specified file, if any, are diplayed when someone connects to the server. + // If set, it overrides the banner string provided by the banner option + BannerFile string `json:"banner_file" mapstructure:"banner_file"` + // If files containing a certificate and matching private key for the server are provided the server will accept + // both plain FTP an explicit FTP over TLS. + // Certificate and key files can be reloaded on demand sending a "SIGHUP" signal on Unix based systems and a + // "paramchange" request to the running service on Windows. + CertificateFile string `json:"certificate_file" mapstructure:"certificate_file"` + CertificateKeyFile string `json:"certificate_key_file" mapstructure:"certificate_key_file"` + // Do not impose the port 20 for active data transfer. Enabling this option allows to run SFTPGo with less privilege + ActiveTransfersPortNon20 bool `json:"active_transfers_port_non_20" mapstructure:"active_transfers_port_non_20"` + // Port Range for data connections. Random if not specified + PassivePortRange PortRange `json:"passive_port_range" mapstructure:"passive_port_range"` +} + +// Initialize configures and starts the FTP server +func (c *Configuration) Initialize(configDir string) error { + var err error + logger.Debug(logSender, "", "initializing FTP server with config %+v", c) + server, err = NewServer(c, configDir) + if err != nil { + return err + } + ftpServer := ftpserver.NewFtpServer(server) + return ftpServer.ListenAndServe() +} + +// ReloadTLSCertificate reloads the TLS certificate and key from the configured paths +func ReloadTLSCertificate() error { + if server != nil && server.certMgr != nil { + return server.certMgr.LoadCertificate(logSender) + } + return nil +} + +func getConfigPath(name, configDir string) string { + if !utils.IsFileInputValid(name) { + return "" + } + if len(name) > 0 && !filepath.IsAbs(name) { + return filepath.Join(configDir, name) + } + return name +} diff --git a/ftpd/ftpd_test.go b/ftpd/ftpd_test.go new file mode 100644 index 00000000..d3ef5e4d --- /dev/null +++ b/ftpd/ftpd_test.go @@ -0,0 +1,1263 @@ +package ftpd_test + +import ( + "crypto/rand" + "crypto/tls" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "os" + "os/exec" + "path" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/jlaffaye/ftp" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + + "github.com/drakkan/sftpgo/common" + "github.com/drakkan/sftpgo/config" + "github.com/drakkan/sftpgo/dataprovider" + "github.com/drakkan/sftpgo/ftpd" + "github.com/drakkan/sftpgo/httpd" + "github.com/drakkan/sftpgo/logger" + "github.com/drakkan/sftpgo/vfs" +) + +const ( + logSender = "ftpdTesting" + ftpServerAddr = "127.0.0.1:2121" + defaultUsername = "test_user_ftp" + defaultPassword = "test_password" + configDir = ".." + osWindows = "windows" + ftpsCert = `-----BEGIN CERTIFICATE----- +MIICHTCCAaKgAwIBAgIUHnqw7QnB1Bj9oUsNpdb+ZkFPOxMwCgYIKoZIzj0EAwIw +RTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGElu +dGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMDAyMDQwOTUzMDRaFw0zMDAyMDEw +OTUzMDRaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYD +VQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwdjAQBgcqhkjOPQIBBgUrgQQA +IgNiAARCjRMqJ85rzMC998X5z761nJ+xL3bkmGVqWvrJ51t5OxV0v25NsOgR82CA +NXUgvhVYs7vNFN+jxtb2aj6Xg+/2G/BNxkaFspIVCzgWkxiz7XE4lgUwX44FCXZM +3+JeUbKjUzBRMB0GA1UdDgQWBBRhLw+/o3+Z02MI/d4tmaMui9W16jAfBgNVHSME +GDAWgBRhLw+/o3+Z02MI/d4tmaMui9W16jAPBgNVHRMBAf8EBTADAQH/MAoGCCqG +SM49BAMCA2kAMGYCMQDqLt2lm8mE+tGgtjDmtFgdOcI72HSbRQ74D5rYTzgST1rY +/8wTi5xl8TiFUyLMUsICMQC5ViVxdXbhuG7gX6yEqSkMKZICHpO8hqFwOD/uaFVI +dV4vKmHUzwK/eIx+8Ay3neE= +-----END CERTIFICATE-----` + ftpsKey = `-----BEGIN EC PARAMETERS----- +BgUrgQQAIg== +-----END EC PARAMETERS----- +-----BEGIN EC PRIVATE KEY----- +MIGkAgEBBDCfMNsN6miEE3rVyUPwElfiJSWaR5huPCzUenZOfJT04GAcQdWvEju3 +UM2lmBLIXpGgBwYFK4EEACKhZANiAARCjRMqJ85rzMC998X5z761nJ+xL3bkmGVq +WvrJ51t5OxV0v25NsOgR82CANXUgvhVYs7vNFN+jxtb2aj6Xg+/2G/BNxkaFspIV +CzgWkxiz7XE4lgUwX44FCXZM3+JeUbI= +-----END EC PRIVATE KEY-----` +) + +var ( + allPerms = []string{dataprovider.PermAny} + homeBasePath string + hookCmdPath string + extAuthPath string + preLoginPath string + logFilePath string +) + +func TestMain(m *testing.M) { + logFilePath = filepath.Join(configDir, "sftpgo_ftpd_test.log") + bannerFileName := "banner_file" + bannerFile := filepath.Join(configDir, bannerFileName) + logger.InitLogger(logFilePath, 5, 1, 28, false, zerolog.DebugLevel) + logger.DebugToConsole("aaa %v", bannerFile) + err := ioutil.WriteFile(bannerFile, []byte("SFTPGo test ready\nsimple banner line\n"), os.ModePerm) + if err != nil { + logger.ErrorToConsole("error creating banner file: %v", err) + } + err = config.LoadConfig(configDir, "") + if err != nil { + logger.ErrorToConsole("error loading configuration: %v", err) + os.Exit(1) + } + providerConf := config.GetProviderConf() + logger.InfoToConsole("Starting FTPD tests, provider: %v", providerConf.Driver) + + commonConf := config.GetCommonConfig() + // we run the test cases with UploadMode atomic and resume support. The non atomic code path + // simply does not execute some code so if it works in atomic mode will + // work in non atomic mode too + commonConf.UploadMode = 2 + homeBasePath = os.TempDir() + if runtime.GOOS != osWindows { + commonConf.Actions.ExecuteOn = []string{"download", "upload", "rename", "delete"} + commonConf.Actions.Hook = hookCmdPath + hookCmdPath, err = exec.LookPath("true") + if err != nil { + logger.Warn(logSender, "", "unable to get hook command: %v", err) + logger.WarnToConsole("unable to get hook command: %v", err) + } + } + + certPath := filepath.Join(os.TempDir(), "test.crt") + keyPath := filepath.Join(os.TempDir(), "test.key") + err = ioutil.WriteFile(certPath, []byte(ftpsCert), os.ModePerm) + if err != nil { + logger.ErrorToConsole("error writing FTPS certificate: %v", err) + os.Exit(1) + } + err = ioutil.WriteFile(keyPath, []byte(ftpsKey), os.ModePerm) + if err != nil { + logger.ErrorToConsole("error writing FTPS private key: %v", err) + os.Exit(1) + } + + common.Initialize(commonConf) + + err = dataprovider.Initialize(providerConf, configDir) + if err != nil { + logger.ErrorToConsole("error initializing data provider: %v", err) + os.Exit(1) + } + + httpConfig := config.GetHTTPConfig() + httpConfig.Initialize(configDir) + + httpdConf := config.GetHTTPDConfig() + httpdConf.BindPort = 8079 + httpd.SetBaseURLAndCredentials("http://127.0.0.1:8079", "", "") + + ftpdConf := config.GetFTPDConfig() + ftpdConf.BindPort = 2121 + ftpdConf.PassivePortRange.Start = 0 + ftpdConf.PassivePortRange.End = 0 + ftpdConf.BannerFile = bannerFileName + ftpdConf.CertificateFile = certPath + ftpdConf.CertificateKeyFile = keyPath + + extAuthPath = filepath.Join(homeBasePath, "extauth.sh") + preLoginPath = filepath.Join(homeBasePath, "prelogin.sh") + + go func() { + logger.Debug(logSender, "", "initializing FTP server with config %+v", ftpdConf) + if err := ftpdConf.Initialize(configDir); err != nil { + logger.ErrorToConsole("could not start FTP server: %v", err) + os.Exit(1) + } + }() + + go func() { + if err := httpdConf.Initialize(configDir, false); err != nil { + logger.ErrorToConsole("could not start HTTP server: %v", err) + os.Exit(1) + } + }() + + waitTCPListening(fmt.Sprintf("%s:%d", ftpdConf.BindAddress, ftpdConf.BindPort)) + waitTCPListening(fmt.Sprintf("%s:%d", httpdConf.BindAddress, httpdConf.BindPort)) + ftpd.ReloadTLSCertificate() //nolint:errcheck + + exitCode := m.Run() + os.Remove(logFilePath) + os.Remove(bannerFile) + os.Remove(extAuthPath) + os.Remove(preLoginPath) + os.Remove(certPath) + os.Remove(keyPath) + os.Exit(exitCode) +} + +func TestBasicFTPHandling(t *testing.T) { + u := getTestUser() + u.QuotaSize = 6553600 + user, _, err := httpd.AddUser(u, http.StatusOK) + assert.NoError(t, err) + client, err := getFTPClient(user, true) + if assert.NoError(t, err) { + assert.Len(t, common.Connections.GetStats(), 1) + testFileName := "test_file.dat" //nolint:goconst + testFilePath := filepath.Join(homeBasePath, testFileName) + testFileSize := int64(65535) + expectedQuotaSize := user.UsedQuotaSize + testFileSize + expectedQuotaFiles := user.UsedQuotaFiles + 1 + err = createTestFile(testFilePath, testFileSize) + assert.NoError(t, err) + + err = checkBasicFTP(client) + assert.NoError(t, err) + err = ftpUploadFile(testFilePath, path.Join("/missing_dir", testFileName), testFileSize, client, 0) + assert.Error(t, err) + err = ftpUploadFile(testFilePath, testFileName, testFileSize, client, 0) + assert.NoError(t, err) + // overwrite an existing file + err = ftpUploadFile(testFilePath, testFileName, testFileSize, client, 0) + assert.NoError(t, err) + localDownloadPath := filepath.Join(homeBasePath, "test_download.dat") + err = ftpDownloadFile(testFileName, localDownloadPath, testFileSize, client, 0) + assert.NoError(t, err) + user, _, err = httpd.GetUserByID(user.ID, http.StatusOK) + assert.NoError(t, err) + assert.Equal(t, expectedQuotaFiles, user.UsedQuotaFiles) + assert.Equal(t, expectedQuotaSize, user.UsedQuotaSize) + err = client.Rename(testFileName, testFileName+"1") + assert.NoError(t, err) + err = client.Delete(testFileName) + assert.Error(t, err) + err = client.Delete(testFileName + "1") + assert.NoError(t, err) + user, _, err = httpd.GetUserByID(user.ID, http.StatusOK) + assert.NoError(t, err) + assert.Equal(t, expectedQuotaFiles-1, user.UsedQuotaFiles) + assert.Equal(t, expectedQuotaSize-testFileSize, user.UsedQuotaSize) + curDir, err := client.CurrentDir() + if assert.NoError(t, err) { + assert.Equal(t, "/", curDir) + } + testDir := "testDir" + err = client.MakeDir(testDir) + assert.NoError(t, err) + err = client.ChangeDir(testDir) + assert.NoError(t, err) + curDir, err = client.CurrentDir() + if assert.NoError(t, err) { + assert.Equal(t, path.Join("/", testDir), curDir) + } + err = ftpUploadFile(testFilePath, testFileName, testFileSize, client, 0) + assert.NoError(t, err) + size, err := client.FileSize(path.Join("/", testDir, testFileName)) + assert.NoError(t, err) + assert.Equal(t, testFileSize, size) + err = client.ChangeDirToParent() + assert.NoError(t, err) + curDir, err = client.CurrentDir() + if assert.NoError(t, err) { + assert.Equal(t, "/", curDir) + } + err = client.Delete(path.Join("/", testDir, testFileName)) + assert.NoError(t, err) + err = client.RemoveDir(testDir) + assert.NoError(t, err) + + err = os.Remove(testFilePath) + assert.NoError(t, err) + err = os.Remove(localDownloadPath) + assert.NoError(t, err) + err = client.Quit() + assert.NoError(t, err) + } + _, err = httpd.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) + assert.Eventually(t, func() bool { return len(common.Connections.GetStats()) == 0 }, 1*time.Second, 50*time.Millisecond) +} + +func TestLoginInvalidPwd(t *testing.T) { + u := getTestUser() + user, _, err := httpd.AddUser(u, http.StatusOK) + assert.NoError(t, err) + user.Password = "wrong" + _, err = getFTPClient(user, false) + assert.Error(t, err) + _, err = httpd.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) +} + +func TestLoginExternalAuth(t *testing.T) { + if runtime.GOOS == osWindows { + t.Skip("this test is not available on Windows") + } + u := getTestUser() + err := dataprovider.Close() + assert.NoError(t, err) + err = config.LoadConfig(configDir, "") + assert.NoError(t, err) + providerConf := config.GetProviderConf() + err = ioutil.WriteFile(extAuthPath, getExtAuthScriptContent(u, false, ""), 0755) + assert.NoError(t, err) + providerConf.ExternalAuthHook = extAuthPath + providerConf.ExternalAuthScope = 0 + err = dataprovider.Initialize(providerConf, configDir) + assert.NoError(t, err) + client, err := getFTPClient(u, true) + if assert.NoError(t, err) { + err = checkBasicFTP(client) + assert.NoError(t, err) + err := client.Quit() + assert.NoError(t, err) + } + u.Username = defaultUsername + "1" + client, err = getFTPClient(u, true) + if !assert.Error(t, err) { + err := client.Quit() + assert.NoError(t, err) + } + + users, _, err := httpd.GetUsers(0, 0, defaultUsername, http.StatusOK) + assert.NoError(t, err) + if assert.Len(t, users, 1) { + user := users[0] + _, err = httpd.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) + } + err = dataprovider.Close() + assert.NoError(t, err) + err = config.LoadConfig(configDir, "") + assert.NoError(t, err) + providerConf = config.GetProviderConf() + err = dataprovider.Initialize(providerConf, configDir) + assert.NoError(t, err) + err = os.Remove(extAuthPath) + assert.NoError(t, err) +} + +func TestPreLoginHook(t *testing.T) { + if runtime.GOOS == osWindows { + t.Skip("this test is not available on Windows") + } + u := getTestUser() + err := dataprovider.Close() + assert.NoError(t, err) + err = config.LoadConfig(configDir, "") + assert.NoError(t, err) + providerConf := config.GetProviderConf() + err = ioutil.WriteFile(preLoginPath, getPreLoginScriptContent(u, false), 0755) + assert.NoError(t, err) + providerConf.PreLoginHook = preLoginPath + err = dataprovider.Initialize(providerConf, configDir) + assert.NoError(t, err) + users, _, err := httpd.GetUsers(0, 0, defaultUsername, http.StatusOK) + assert.NoError(t, err) + assert.Equal(t, 0, len(users)) + client, err := getFTPClient(u, false) + if assert.NoError(t, err) { + err = checkBasicFTP(client) + assert.NoError(t, err) + err := client.Quit() + assert.NoError(t, err) + } + + users, _, err = httpd.GetUsers(0, 0, defaultUsername, http.StatusOK) + assert.NoError(t, err) + assert.Equal(t, 1, len(users)) + user := users[0] + + // test login with an existing user + client, err = getFTPClient(user, true) + if assert.NoError(t, err) { + err = checkBasicFTP(client) + assert.NoError(t, err) + err := client.Quit() + assert.NoError(t, err) + } + + err = ioutil.WriteFile(preLoginPath, getPreLoginScriptContent(user, true), 0755) + assert.NoError(t, err) + client, err = getFTPClient(u, false) + if !assert.Error(t, err) { + err := client.Quit() + assert.NoError(t, err) + } + user.Status = 0 + err = ioutil.WriteFile(preLoginPath, getPreLoginScriptContent(user, false), 0755) + assert.NoError(t, err) + client, err = getFTPClient(u, false) + if !assert.Error(t, err, "pre-login script returned a disabled user, login must fail") { + err := client.Quit() + assert.NoError(t, err) + } + + _, err = httpd.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) + err = dataprovider.Close() + assert.NoError(t, err) + err = config.LoadConfig(configDir, "") + assert.NoError(t, err) + providerConf = config.GetProviderConf() + err = dataprovider.Initialize(providerConf, configDir) + assert.NoError(t, err) + err = os.Remove(preLoginPath) + assert.NoError(t, err) +} + +func TestMaxSessions(t *testing.T) { + u := getTestUser() + u.MaxSessions = 1 + user, _, err := httpd.AddUser(u, http.StatusOK) + assert.NoError(t, err) + client, err := getFTPClient(user, true) + if assert.NoError(t, err) { + defer func() { + err := client.Quit() + assert.NoError(t, err) + }() + err = checkBasicFTP(client) + assert.NoError(t, err) + _, err = getFTPClient(user, false) + assert.Error(t, err) + } + _, err = httpd.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) +} + +func TestZeroBytesTransfers(t *testing.T) { + u := getTestUser() + user, _, err := httpd.AddUser(u, http.StatusOK) + assert.NoError(t, err) + for _, useTLS := range []bool{true, false} { + client, err := getFTPClient(user, useTLS) + if assert.NoError(t, err) { + testFileName := "testfilename" + err = checkBasicFTP(client) + assert.NoError(t, err) + localDownloadPath := filepath.Join(homeBasePath, "empty_download") + err = ioutil.WriteFile(localDownloadPath, []byte(""), os.ModePerm) + assert.NoError(t, err) + err = ftpUploadFile(localDownloadPath, testFileName, 0, client, 0) + assert.NoError(t, err) + size, err := client.FileSize(testFileName) + assert.NoError(t, err) + assert.Equal(t, int64(0), size) + err = os.Remove(localDownloadPath) + assert.NoError(t, err) + assert.NoFileExists(t, localDownloadPath) + err = ftpDownloadFile(testFileName, localDownloadPath, 0, client, 0) + assert.NoError(t, err) + assert.FileExists(t, localDownloadPath) + err = client.Quit() + assert.NoError(t, err) + err = os.Remove(localDownloadPath) + assert.NoError(t, err) + } + } + _, err = httpd.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) +} + +func TestDownloadsError(t *testing.T) { + u := getTestUser() + u.QuotaFiles = 1 + subDir1 := "sub1" + subDir2 := "sub2" + u.Permissions[path.Join("/", subDir1)] = []string{dataprovider.PermListItems} + u.Permissions[path.Join("/", subDir2)] = []string{dataprovider.PermListItems, dataprovider.PermUpload, + dataprovider.PermDelete, dataprovider.PermDownload} + u.Filters.FileExtensions = []dataprovider.ExtensionsFilter{ + { + Path: "/sub2", + AllowedExtensions: []string{}, + DeniedExtensions: []string{".zip"}, + }, + } + user, _, err := httpd.AddUser(u, http.StatusOK) + assert.NoError(t, err) + client, err := getFTPClient(user, true) + if assert.NoError(t, err) { + testFilePath1 := filepath.Join(user.HomeDir, subDir1, "file.zip") + testFilePath2 := filepath.Join(user.HomeDir, subDir2, "file.zip") + err = os.MkdirAll(filepath.Dir(testFilePath1), os.ModePerm) + assert.NoError(t, err) + err = os.MkdirAll(filepath.Dir(testFilePath2), os.ModePerm) + assert.NoError(t, err) + err = ioutil.WriteFile(testFilePath1, []byte("file1"), os.ModePerm) + assert.NoError(t, err) + err = ioutil.WriteFile(testFilePath2, []byte("file2"), os.ModePerm) + assert.NoError(t, err) + localDownloadPath := filepath.Join(homeBasePath, "test_download.dat") + err = ftpDownloadFile(path.Join("/", subDir1, "file.zip"), localDownloadPath, 5, client, 0) + assert.Error(t, err) + err = ftpDownloadFile(path.Join("/", subDir2, "file.zip"), localDownloadPath, 5, client, 0) + assert.Error(t, err) + err = ftpDownloadFile("/missing.zip", localDownloadPath, 5, client, 0) + assert.Error(t, err) + err = client.Quit() + assert.NoError(t, err) + err = os.Remove(localDownloadPath) + assert.NoError(t, err) + } + _, err = httpd.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) +} + +func TestUploadErrors(t *testing.T) { + u := getTestUser() + u.QuotaSize = 65535 + subDir1 := "sub1" + subDir2 := "sub2" + u.Permissions[path.Join("/", subDir1)] = []string{dataprovider.PermListItems} + u.Permissions[path.Join("/", subDir2)] = []string{dataprovider.PermListItems, dataprovider.PermUpload, + dataprovider.PermDelete} + u.Filters.FileExtensions = []dataprovider.ExtensionsFilter{ + { + Path: "/sub2", + AllowedExtensions: []string{}, + DeniedExtensions: []string{".zip"}, + }, + } + user, _, err := httpd.AddUser(u, http.StatusOK) + assert.NoError(t, err) + client, err := getFTPClient(user, true) + if assert.NoError(t, err) { + testFileName := "test_file.dat" + testFilePath := filepath.Join(homeBasePath, testFileName) + testFileSize := user.QuotaSize + err = createTestFile(testFilePath, testFileSize) + assert.NoError(t, err) + err = client.MakeDir(subDir1) + assert.NoError(t, err) + err = client.MakeDir(subDir2) + assert.NoError(t, err) + err = client.ChangeDir(subDir1) + assert.NoError(t, err) + err = ftpUploadFile(testFilePath, testFileName, testFileSize, client, 0) + assert.Error(t, err) + err = client.ChangeDirToParent() + assert.NoError(t, err) + err = client.ChangeDir(subDir2) + assert.NoError(t, err) + err = ftpUploadFile(testFilePath, testFileName+".zip", testFileSize, client, 0) + assert.Error(t, err) + err = ftpUploadFile(testFilePath, testFileName, testFileSize, client, 0) + assert.NoError(t, err) + err = ftpUploadFile(testFilePath, testFileName, testFileSize, client, 0) + assert.Error(t, err) + err = client.ChangeDir("/") + assert.NoError(t, err) + err = ftpUploadFile(testFilePath, subDir1, testFileSize, client, 0) + assert.Error(t, err) + // overquota + err = ftpUploadFile(testFilePath, testFileName+"1", testFileSize, client, 0) + assert.Error(t, err) + err = client.Delete(path.Join("/", subDir2, testFileName)) + assert.NoError(t, err) + err = ftpUploadFile(testFilePath, testFileName, testFileSize, client, 0) + assert.NoError(t, err) + err = ftpUploadFile(testFilePath, testFileName, testFileSize, client, 0) + assert.Error(t, err) + err = client.Quit() + assert.NoError(t, err) + } + _, err = httpd.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) +} + +func TestResume(t *testing.T) { + u := getTestUser() + user, _, err := httpd.AddUser(u, http.StatusOK) + assert.NoError(t, err) + client, err := getFTPClient(user, true) + if assert.NoError(t, err) { + testFileName := "test_file.dat" + testFilePath := filepath.Join(homeBasePath, testFileName) + data := []byte("test data") + err = ioutil.WriteFile(testFilePath, data, os.ModePerm) + assert.NoError(t, err) + err = ftpUploadFile(testFilePath, testFileName, int64(len(data)), client, 0) + assert.NoError(t, err) + err = ftpUploadFile(testFilePath, testFileName, int64(len(data)+5), client, 5) + assert.NoError(t, err) + readed, err := ioutil.ReadFile(filepath.Join(user.GetHomeDir(), testFileName)) + assert.NoError(t, err) + assert.Equal(t, "test test data", string(readed)) + localDownloadPath := filepath.Join(homeBasePath, "test_download.dat") + err = ftpDownloadFile(testFileName, localDownloadPath, int64(len(data)), client, 5) + assert.NoError(t, err) + readed, err = ioutil.ReadFile(localDownloadPath) + assert.NoError(t, err) + assert.Equal(t, data, readed) + err = client.Delete(testFileName) + assert.NoError(t, err) + err = ftpUploadFile(testFilePath, testFileName, int64(len(data)), client, 0) + assert.NoError(t, err) + // now append to a file + srcFile, err := os.Open(testFilePath) + if assert.NoError(t, err) { + err = client.Append(testFileName, srcFile) + assert.NoError(t, err) + err = srcFile.Close() + assert.NoError(t, err) + size, err := client.FileSize(testFileName) + assert.NoError(t, err) + assert.Equal(t, int64(2*len(data)), size) + err = ftpDownloadFile(testFileName, localDownloadPath, int64(2*len(data)), client, 0) + assert.NoError(t, err) + readed, err = ioutil.ReadFile(localDownloadPath) + assert.NoError(t, err) + expected := append(data, data...) + assert.Equal(t, expected, readed) + } + err = client.Quit() + assert.NoError(t, err) + err = os.Remove(testFilePath) + assert.NoError(t, err) + err = os.Remove(localDownloadPath) + assert.NoError(t, err) + } + _, err = httpd.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) +} + +func TestQuotaLimits(t *testing.T) { + u := getTestUser() + u.QuotaFiles = 1 + user, _, err := httpd.AddUser(u, http.StatusOK) + assert.NoError(t, err) + testFileSize := int64(65535) + testFileName := "test_file.dat" + testFilePath := filepath.Join(homeBasePath, testFileName) + err = createTestFile(testFilePath, testFileSize) + assert.NoError(t, err) + testFileSize1 := int64(131072) + testFileName1 := "test_file1.dat" + testFilePath1 := filepath.Join(homeBasePath, testFileName1) + err = createTestFile(testFilePath1, testFileSize1) + assert.NoError(t, err) + testFileSize2 := int64(32768) + testFileName2 := "test_file2.dat" + testFilePath2 := filepath.Join(homeBasePath, testFileName2) + err = createTestFile(testFilePath2, testFileSize2) + assert.NoError(t, err) + // test quota files + client, err := getFTPClient(user, false) + if assert.NoError(t, err) { + err = ftpUploadFile(testFilePath, testFileName+".quota", testFileSize, client, 0) + assert.NoError(t, err) + err = ftpUploadFile(testFilePath, testFileName+".quota1", testFileSize, client, 0) + assert.Error(t, err) + err = client.Rename(testFileName+".quota", testFileName) + assert.NoError(t, err) + err = client.Quit() + assert.NoError(t, err) + } + // test quota size + user.QuotaSize = testFileSize - 1 + user.QuotaFiles = 0 + user, _, err = httpd.UpdateUser(user, http.StatusOK) + assert.NoError(t, err) + client, err = getFTPClient(user, true) + if assert.NoError(t, err) { + err = ftpUploadFile(testFilePath, testFileName+".quota", testFileSize, client, 0) + assert.Error(t, err) + err = client.Rename(testFileName, testFileName+".quota") + assert.NoError(t, err) + err = client.Quit() + assert.NoError(t, err) + } + // now test quota limits while uploading the current file, we have 1 bytes remaining + user.QuotaSize = testFileSize + 1 + user.QuotaFiles = 0 + user, _, err = httpd.UpdateUser(user, http.StatusOK) + assert.NoError(t, err) + client, err = getFTPClient(user, false) + if assert.NoError(t, err) { + err = ftpUploadFile(testFilePath1, testFileName1, testFileSize1, client, 0) + assert.Error(t, err) + _, err = client.FileSize(testFileName1) + assert.Error(t, err) + err = client.Rename(testFileName+".quota", testFileName) + assert.NoError(t, err) + // overwriting an existing file will work if the resulting size is lesser or equal than the current one + err = ftpUploadFile(testFilePath, testFileName, testFileSize, client, 0) + assert.NoError(t, err) + err = ftpUploadFile(testFilePath2, testFileName, testFileSize2, client, 0) + assert.NoError(t, err) + err = ftpUploadFile(testFilePath1, testFileName, testFileSize1, client, 0) + assert.Error(t, err) + err = ftpUploadFile(testFilePath1, testFileName, testFileSize1, client, 10) + assert.Error(t, err) + err = ftpUploadFile(testFilePath2, testFileName, testFileSize2, client, 0) + assert.NoError(t, err) + err = client.Quit() + assert.NoError(t, err) + } + + err = os.Remove(testFilePath) + assert.NoError(t, err) + err = os.Remove(testFilePath1) + assert.NoError(t, err) + err = os.Remove(testFilePath2) + assert.NoError(t, err) + _, err = httpd.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) +} + +func TestLoginWithIPilters(t *testing.T) { + u := getTestUser() + u.Filters.DeniedIP = []string{"192.167.0.0/24", "172.18.0.0/16"} + u.Filters.AllowedIP = []string{"172.19.0.0/16"} + user, _, err := httpd.AddUser(u, http.StatusOK) + assert.NoError(t, err) + client, err := getFTPClient(user, true) + if !assert.Error(t, err) { + err = client.Quit() + assert.NoError(t, err) + } + + _, err = httpd.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) +} + +func TestLoginInvalidFs(t *testing.T) { + u := getTestUser() + u.FsConfig.Provider = 2 + u.FsConfig.GCSConfig.Bucket = "test" + u.FsConfig.GCSConfig.Credentials = base64.StdEncoding.EncodeToString([]byte("invalid JSON for credentials")) + user, _, err := httpd.AddUser(u, http.StatusOK) + assert.NoError(t, err) + // now remove the credentials file so the filesystem creation will fail + providerConf := config.GetProviderConf() + credentialsFile := filepath.Join(providerConf.CredentialsPath, fmt.Sprintf("%v_gcs_credentials.json", u.Username)) + if !filepath.IsAbs(credentialsFile) { + credentialsFile = filepath.Join(configDir, credentialsFile) + } + err = os.Remove(credentialsFile) + assert.NoError(t, err) + client, err := getFTPClient(user, false) + if !assert.Error(t, err) { + err = client.Quit() + assert.NoError(t, err) + } + _, err = httpd.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) +} + +func TestClientClose(t *testing.T) { + u := getTestUser() + user, _, err := httpd.AddUser(u, http.StatusOK) + assert.NoError(t, err) + client, err := getFTPClient(user, true) + if assert.NoError(t, err) { + err = checkBasicFTP(client) + assert.NoError(t, err) + stats := common.Connections.GetStats() + if assert.Len(t, stats, 1) { + common.Connections.Close(stats[0].ConnectionID) + assert.Eventually(t, func() bool { return len(common.Connections.GetStats()) == 0 }, + 1*time.Second, 50*time.Millisecond) + } + } + _, err = httpd.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) +} + +func TestRename(t *testing.T) { + u := getTestUser() + user, _, err := httpd.AddUser(u, http.StatusOK) + assert.NoError(t, err) + testDir := "adir" + testFileName := "test_file.dat" + testFilePath := filepath.Join(homeBasePath, testFileName) + testFileSize := int64(65535) + err = createTestFile(testFilePath, testFileSize) + assert.NoError(t, err) + client, err := getFTPClient(user, false) + if assert.NoError(t, err) { + assert.NoError(t, err) + err = checkBasicFTP(client) + assert.NoError(t, err) + err = ftpUploadFile(testFilePath, testFileName, testFileSize, client, 0) + assert.NoError(t, err) + err = client.MakeDir(testDir) + assert.NoError(t, err) + err = client.Rename(testFileName, path.Join("missing", testFileName)) + assert.Error(t, err) + err = client.Rename(testFileName, path.Join(testDir, testFileName)) + assert.NoError(t, err) + size, err := client.FileSize(path.Join(testDir, testFileName)) + assert.NoError(t, err) + assert.Equal(t, testFileSize, size) + if runtime.GOOS != osWindows { + otherDir := "dir" + err = client.MakeDir(otherDir) + assert.NoError(t, err) + err = client.MakeDir(path.Join(otherDir, testDir)) + assert.NoError(t, err) + code, response, err := client.SendCustomCommand(fmt.Sprintf("SITE CHMOD 0001 %v", otherDir)) + assert.NoError(t, err) + assert.Equal(t, ftp.StatusCommandOK, code) + assert.Equal(t, "SITE CHMOD command successful", response) + err = client.Rename(testDir, path.Join(otherDir, testDir)) + assert.Error(t, err) + + code, response, err = client.SendCustomCommand(fmt.Sprintf("SITE CHMOD 755 %v", otherDir)) + assert.NoError(t, err) + assert.Equal(t, ftp.StatusCommandOK, code) + assert.Equal(t, "SITE CHMOD command successful", response) + } + err = client.Quit() + assert.NoError(t, err) + } + user.Permissions[path.Join("/", testDir)] = []string{dataprovider.PermListItems} + user, _, err = httpd.UpdateUser(user, http.StatusOK) + assert.NoError(t, err) + client, err = getFTPClient(user, false) + if assert.NoError(t, err) { + defer func() { + err := client.Quit() + assert.NoError(t, err) + }() + + err = client.Rename(path.Join(testDir, testFileName), testFileName) + assert.Error(t, err) + } + + err = os.Remove(testFilePath) + assert.NoError(t, err) + _, err = httpd.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) +} + +func TestStat(t *testing.T) { + u := getTestUser() + u.Permissions["/subdir"] = []string{dataprovider.PermUpload} + user, _, err := httpd.AddUser(u, http.StatusOK) + assert.NoError(t, err) + client, err := getFTPClient(user, false) + if assert.NoError(t, err) { + defer func() { + err := client.Quit() + assert.NoError(t, err) + }() + + subDir := "subdir" + testFileName := "test_file.dat" + testFilePath := filepath.Join(homeBasePath, testFileName) + testFileSize := int64(65535) + err = createTestFile(testFilePath, testFileSize) + assert.NoError(t, err) + err = client.MakeDir(subDir) + assert.NoError(t, err) + err = ftpUploadFile(testFilePath, testFileName, testFileSize, client, 0) + assert.NoError(t, err) + err = ftpUploadFile(testFilePath, path.Join("/", subDir, testFileName), testFileSize, client, 0) + assert.Error(t, err) + size, err := client.FileSize(testFileName) + assert.NoError(t, err) + assert.Equal(t, testFileSize, size) + _, err = client.FileSize(path.Join("/", subDir, testFileName)) + assert.Error(t, err) + _, err = client.FileSize("missing file") + assert.Error(t, err) + + err = os.Remove(testFilePath) + assert.NoError(t, err) + } + + _, err = httpd.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) +} + +func TestUploadOverwriteVfolder(t *testing.T) { + u := getTestUser() + vdir := "/vdir" + mappedPath := filepath.Join(os.TempDir(), "vdir") + u.VirtualFolders = append(u.VirtualFolders, vfs.VirtualFolder{ + BaseVirtualFolder: vfs.BaseVirtualFolder{ + MappedPath: mappedPath, + }, + VirtualPath: vdir, + QuotaSize: -1, + QuotaFiles: -1, + }) + err := os.MkdirAll(mappedPath, os.ModePerm) + assert.NoError(t, err) + user, _, err := httpd.AddUser(u, http.StatusOK) + assert.NoError(t, err) + client, err := getFTPClient(user, false) + if assert.NoError(t, err) { + testFileName := "test_file.dat" + testFilePath := filepath.Join(homeBasePath, testFileName) + testFileSize := int64(65535) + err = createTestFile(testFilePath, testFileSize) + assert.NoError(t, err) + err = ftpUploadFile(testFilePath, path.Join(vdir, testFileName), testFileSize, client, 0) + assert.NoError(t, err) + folder, _, err := httpd.GetFolders(0, 0, mappedPath, http.StatusOK) + assert.NoError(t, err) + if assert.Len(t, folder, 1) { + f := folder[0] + assert.Equal(t, testFileSize, f.UsedQuotaSize) + assert.Equal(t, 1, f.UsedQuotaFiles) + } + err = ftpUploadFile(testFilePath, path.Join(vdir, testFileName), testFileSize, client, 0) + assert.NoError(t, err) + folder, _, err = httpd.GetFolders(0, 0, mappedPath, http.StatusOK) + assert.NoError(t, err) + if assert.Len(t, folder, 1) { + f := folder[0] + assert.Equal(t, testFileSize, f.UsedQuotaSize) + assert.Equal(t, 1, f.UsedQuotaFiles) + } + err = client.Quit() + assert.NoError(t, err) + err = os.Remove(testFilePath) + assert.NoError(t, err) + } + _, err = httpd.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) + _, err = httpd.RemoveFolder(vfs.BaseVirtualFolder{MappedPath: mappedPath}, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) + err = os.RemoveAll(mappedPath) + assert.NoError(t, err) +} + +func TestAllocate(t *testing.T) { + u := getTestUser() + mappedPath := filepath.Join(os.TempDir(), "vdir") + u.VirtualFolders = append(u.VirtualFolders, vfs.VirtualFolder{ + BaseVirtualFolder: vfs.BaseVirtualFolder{ + MappedPath: mappedPath, + }, + VirtualPath: "/vdir", + QuotaSize: 110, + }) + err := os.MkdirAll(mappedPath, os.ModePerm) + assert.NoError(t, err) + user, _, err := httpd.AddUser(u, http.StatusOK) + assert.NoError(t, err) + client, err := getFTPClient(user, false) + if assert.NoError(t, err) { + code, response, err := client.SendCustomCommand("allo 2000000") + assert.NoError(t, err) + assert.Equal(t, ftp.StatusCommandOK, code) + assert.Equal(t, "Done !", response) + err = client.Quit() + assert.NoError(t, err) + } + user.QuotaSize = 100 + user, _, err = httpd.UpdateUser(user, http.StatusOK) + assert.NoError(t, err) + client, err = getFTPClient(user, false) + if assert.NoError(t, err) { + testFileName := "test_file.dat" + testFilePath := filepath.Join(homeBasePath, testFileName) + testFileSize := user.QuotaSize - 1 + err = createTestFile(testFilePath, testFileSize) + assert.NoError(t, err) + code, response, err := client.SendCustomCommand("allo 99") + assert.NoError(t, err) + assert.Equal(t, ftp.StatusCommandOK, code) + assert.Equal(t, "Done !", response) + code, response, err = client.SendCustomCommand("allo 100") + assert.NoError(t, err) + assert.Equal(t, ftp.StatusCommandOK, code) + assert.Equal(t, "Done !", response) + code, response, err = client.SendCustomCommand("allo 150") + assert.NoError(t, err) + assert.Equal(t, ftp.StatusFileUnavailable, code) + assert.Contains(t, response, common.ErrQuotaExceeded.Error()) + + err = ftpUploadFile(testFilePath, testFileName, testFileSize, client, 0) + assert.NoError(t, err) + // we still have space in vdir + code, response, err = client.SendCustomCommand("allo 50") + assert.NoError(t, err) + assert.Equal(t, ftp.StatusCommandOK, code) + assert.Equal(t, "Done !", response) + err = ftpUploadFile(testFilePath, path.Join("/vdir", testFileName), testFileSize, client, 0) + assert.NoError(t, err) + code, response, err = client.SendCustomCommand("allo 50") + assert.NoError(t, err) + assert.Equal(t, ftp.StatusFileUnavailable, code) + assert.Contains(t, response, common.ErrQuotaExceeded.Error()) + + err = client.Quit() + assert.NoError(t, err) + err = os.Remove(testFilePath) + assert.NoError(t, err) + } + + _, err = httpd.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) + _, err = httpd.RemoveFolder(vfs.BaseVirtualFolder{MappedPath: mappedPath}, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) + err = os.RemoveAll(mappedPath) + assert.NoError(t, err) +} + +func TestChtimes(t *testing.T) { + u := getTestUser() + user, _, err := httpd.AddUser(u, http.StatusOK) + assert.NoError(t, err) + client, err := getFTPClient(user, false) + if assert.NoError(t, err) { + defer func() { + err := client.Quit() + assert.NoError(t, err) + }() + + testFileName := "test_file.dat" + testFilePath := filepath.Join(homeBasePath, testFileName) + testFileSize := int64(65535) + err = createTestFile(testFilePath, testFileSize) + assert.NoError(t, err) + err = checkBasicFTP(client) + assert.NoError(t, err) + err = ftpUploadFile(testFilePath, testFileName, testFileSize, client, 0) + assert.NoError(t, err) + + mtime := time.Now().Format("20060102150405") + code, response, err := client.SendCustomCommand(fmt.Sprintf("MFMT %v %v", mtime, testFileName)) + assert.NoError(t, err) + assert.Equal(t, ftp.StatusFile, code) + assert.Equal(t, fmt.Sprintf("Modify=%v; %v", mtime, testFileName), response) + + err = os.Remove(testFilePath) + assert.NoError(t, err) + } + _, err = httpd.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) +} + +func TestChmod(t *testing.T) { + if runtime.GOOS == osWindows { + t.Skip("chmod is partially supported on Windows") + } + u := getTestUser() + user, _, err := httpd.AddUser(u, http.StatusOK) + assert.NoError(t, err) + client, err := getFTPClient(user, true) + if assert.NoError(t, err) { + defer func() { + err := client.Quit() + assert.NoError(t, err) + }() + + testFileName := "test_file.dat" + testFilePath := filepath.Join(homeBasePath, testFileName) + testFileSize := int64(131072) + err = createTestFile(testFilePath, testFileSize) + assert.NoError(t, err) + err = checkBasicFTP(client) + assert.NoError(t, err) + err = ftpUploadFile(testFilePath, testFileName, testFileSize, client, 0) + assert.NoError(t, err) + + code, response, err := client.SendCustomCommand(fmt.Sprintf("SITE CHMOD 600 %v", testFileName)) + assert.NoError(t, err) + assert.Equal(t, ftp.StatusCommandOK, code) + assert.Equal(t, "SITE CHMOD command successful", response) + + fi, err := os.Stat(filepath.Join(user.HomeDir, testFileName)) + if assert.NoError(t, err) { + assert.Equal(t, os.FileMode(0600), fi.Mode().Perm()) + } + + err = os.Remove(testFilePath) + assert.NoError(t, err) + } + _, err = httpd.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) +} + +func checkBasicFTP(client *ftp.ServerConn) error { + _, err := client.CurrentDir() + if err != nil { + return err + } + err = client.NoOp() + if err != nil { + return err + } + _, err = client.List(".") + if err != nil { + return err + } + return nil +} + +func ftpUploadFile(localSourcePath string, remoteDestPath string, expectedSize int64, client *ftp.ServerConn, offset uint64) error { + srcFile, err := os.Open(localSourcePath) + if err != nil { + return err + } + defer srcFile.Close() + if offset > 0 { + err = client.StorFrom(remoteDestPath, srcFile, offset) + } else { + err = client.Stor(remoteDestPath, srcFile) + } + if err != nil { + return err + } + if expectedSize > 0 { + size, err := client.FileSize(remoteDestPath) + if err != nil { + return err + } + if size != expectedSize { + return fmt.Errorf("uploaded file size does not match, actual: %v, expected: %v", size, expectedSize) + } + } + return nil +} + +func ftpDownloadFile(remoteSourcePath string, localDestPath string, expectedSize int64, client *ftp.ServerConn, offset uint64) error { + downloadDest, err := os.Create(localDestPath) + if err != nil { + return err + } + defer downloadDest.Close() + var r *ftp.Response + if offset > 0 { + r, err = client.RetrFrom(remoteSourcePath, offset) + } else { + r, err = client.Retr(remoteSourcePath) + } + if err != nil { + return err + } + defer r.Close() + + written, err := io.Copy(downloadDest, r) + if err != nil { + return err + } + if written != expectedSize { + return fmt.Errorf("downloaded file size does not match, actual: %v, expected: %v", written, expectedSize) + } + return nil +} + +func getFTPClient(user dataprovider.User, useTLS bool) (*ftp.ServerConn, error) { + ftpOptions := []ftp.DialOption{ftp.DialWithTimeout(5 * time.Second)} + if useTLS { + tlsConfig := &tls.Config{ + ServerName: "localhost", + InsecureSkipVerify: true, // use this for tests only + } + ftpOptions = append(ftpOptions, ftp.DialWithExplicitTLS(tlsConfig)) + } + client, err := ftp.Dial(ftpServerAddr, ftpOptions...) + if err != nil { + return nil, err + } + pwd := defaultPassword + if len(user.Password) > 0 { + pwd = user.Password + } + err = client.Login(user.Username, pwd) + if err != nil { + return nil, err + } + return client, err +} + +func waitTCPListening(address string) { + for { + conn, err := net.Dial("tcp", address) + if err != nil { + logger.WarnToConsole("tcp server %v not listening: %v\n", address, err) + time.Sleep(100 * time.Millisecond) + continue + } + logger.InfoToConsole("tcp server %v now listening\n", address) + conn.Close() + break + } +} + +func getTestUser() dataprovider.User { + user := dataprovider.User{ + Username: defaultUsername, + Password: defaultPassword, + HomeDir: filepath.Join(homeBasePath, defaultUsername), + Status: 1, + ExpirationDate: 0, + } + user.Permissions = make(map[string][]string) + user.Permissions["/"] = allPerms + return user +} + +func getExtAuthScriptContent(user dataprovider.User, nonJSONResponse bool, username string) []byte { + extAuthContent := []byte("#!/bin/sh\n\n") + extAuthContent = append(extAuthContent, []byte(fmt.Sprintf("if test \"$SFTPGO_AUTHD_USERNAME\" = \"%v\"; then\n", user.Username))...) + if len(username) > 0 { + user.Username = username + } + u, _ := json.Marshal(user) + if nonJSONResponse { + extAuthContent = append(extAuthContent, []byte("echo 'text response'\n")...) + } else { + extAuthContent = append(extAuthContent, []byte(fmt.Sprintf("echo '%v'\n", string(u)))...) + } + extAuthContent = append(extAuthContent, []byte("else\n")...) + if nonJSONResponse { + extAuthContent = append(extAuthContent, []byte("echo 'text response'\n")...) + } else { + extAuthContent = append(extAuthContent, []byte("echo '{\"username\":\"\"}'\n")...) + } + extAuthContent = append(extAuthContent, []byte("fi\n")...) + return extAuthContent +} + +func getPreLoginScriptContent(user dataprovider.User, nonJSONResponse bool) []byte { + content := []byte("#!/bin/sh\n\n") + if nonJSONResponse { + content = append(content, []byte("echo 'text response'\n")...) + return content + } + if len(user.Username) > 0 { + u, _ := json.Marshal(user) + content = append(content, []byte(fmt.Sprintf("echo '%v'\n", string(u)))...) + } + return content +} + +func createTestFile(path string, size int64) error { + baseDir := filepath.Dir(path) + if _, err := os.Stat(baseDir); os.IsNotExist(err) { + err = os.MkdirAll(baseDir, os.ModePerm) + if err != nil { + return err + } + } + content := make([]byte, size) + _, err := rand.Read(content) + if err != nil { + return err + } + return ioutil.WriteFile(path, content, os.ModePerm) +} diff --git a/ftpd/handler.go b/ftpd/handler.go new file mode 100644 index 00000000..350a6bcc --- /dev/null +++ b/ftpd/handler.go @@ -0,0 +1,387 @@ +package ftpd + +import ( + "errors" + "os" + "path" + "time" + + ftpserver "github.com/fclairamb/ftpserverlib" + "github.com/spf13/afero" + + "github.com/drakkan/sftpgo/common" + "github.com/drakkan/sftpgo/dataprovider" + "github.com/drakkan/sftpgo/logger" + "github.com/drakkan/sftpgo/vfs" +) + +var ( + errNotImplemented = errors.New("Not implemented") +) + +// Connection details for an FTP connection. +// It implements common.ActiveConnection and ftpserver.ClientDriver interfaces +type Connection struct { + *common.BaseConnection + clientContext ftpserver.ClientContext +} + +// GetClientVersion returns the connected client's version. +// It returns "Unknown" if the client does not advertise its +// version +func (c *Connection) GetClientVersion() string { + version := c.clientContext.GetClientVersion() + if len(version) > 0 { + return version + } + return "Unknown" +} + +// GetRemoteAddress return the connected client's address +func (c *Connection) GetRemoteAddress() string { + return c.clientContext.RemoteAddr().String() +} + +// SetConnDeadline does nothing +func (c *Connection) SetConnDeadline() {} + +// Disconnect disconnects the client +func (c *Connection) Disconnect() error { + return c.clientContext.Close(ftpserver.StatusServiceNotAvailable, "connection closed") +} + +// GetCommand returns an empty string +func (c *Connection) GetCommand() string { + return "" +} + +// Create is not implemented we use ClientDriverExtentionFileTransfer +func (c *Connection) Create(name string) (afero.File, error) { + return nil, errNotImplemented +} + +// Mkdir creates a directory using the connection filesystem +func (c *Connection) Mkdir(name string, perm os.FileMode) error { + c.UpdateLastActivity() + + p, err := c.Fs.ResolvePath(name) + if err != nil { + return c.GetFsError(err) + } + return c.CreateDir(p, name) +} + +// MkdirAll is not implemented, we don't need it +func (c *Connection) MkdirAll(path string, perm os.FileMode) error { + return errNotImplemented +} + +// Open is not implemented we use ClientDriverExtentionFileTransfer and ClientDriverExtensionFileList +func (c *Connection) Open(name string) (afero.File, error) { + return nil, errNotImplemented +} + +// OpenFile is not implemented we use ClientDriverExtentionFileTransfer +func (c *Connection) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) { + return nil, errNotImplemented +} + +// Remove removes a file or an empty directory +func (c *Connection) Remove(name string) error { + c.UpdateLastActivity() + + p, err := c.Fs.ResolvePath(name) + if err != nil { + return c.GetFsError(err) + } + + var fi os.FileInfo + if fi, err = c.Fs.Lstat(p); err != nil { + c.Log(logger.LevelWarn, "failed to remove a file %#v: stat error: %+v", p, err) + return c.GetFsError(err) + } + + if fi.IsDir() && fi.Mode()&os.ModeSymlink != os.ModeSymlink { + return c.RemoveDir(p, name) + } + return c.RemoveFile(p, name, fi) +} + +// RemoveAll is not implemented, we don't need it +func (c *Connection) RemoveAll(path string) error { + return errNotImplemented +} + +// Rename renames a file or a directory +func (c *Connection) Rename(oldname, newname string) error { + c.UpdateLastActivity() + + p, err := c.Fs.ResolvePath(oldname) + if err != nil { + return c.GetFsError(err) + } + t, err := c.Fs.ResolvePath(newname) + if err != nil { + return c.GetFsError(err) + } + + if err = c.BaseConnection.Rename(p, t, oldname, newname); err != nil { + return err + } + + vfs.SetPathPermissions(c.Fs, t, c.User.GetUID(), c.User.GetGID()) + return nil +} + +// Stat returns a FileInfo describing the named file/directory, or an error, +// if any happens +func (c *Connection) Stat(name string) (os.FileInfo, error) { + c.UpdateLastActivity() + + if !c.User.HasPerm(dataprovider.PermListItems, path.Dir(name)) { + return nil, c.GetPermissionDeniedError() + } + + p, err := c.Fs.ResolvePath(name) + if err != nil { + return nil, c.GetFsError(err) + } + fi, err := c.Fs.Stat(p) + if err != nil { + c.Log(logger.LevelWarn, "error running stat on path: %+v", err) + return nil, c.GetFsError(err) + } + return fi, nil +} + +// Name returns the name of this connection +func (c *Connection) Name() string { + return c.GetID() +} + +// Chmod changes the mode of the named file/directory +func (c *Connection) Chmod(name string, mode os.FileMode) error { + c.UpdateLastActivity() + + p, err := c.Fs.ResolvePath(name) + if err != nil { + return c.GetFsError(err) + } + attrs := common.StatAttributes{ + Flags: common.StatAttrPerms, + Mode: mode, + } + return c.SetStat(p, name, &attrs) +} + +// Chtimes changes the access and modification times of the named file +func (c *Connection) Chtimes(name string, atime time.Time, mtime time.Time) error { + c.UpdateLastActivity() + + p, err := c.Fs.ResolvePath(name) + if err != nil { + return c.GetFsError(err) + } + attrs := common.StatAttributes{ + Flags: common.StatAttrTimes, + Atime: atime, + Mtime: mtime, + } + return c.SetStat(p, name, &attrs) +} + +// AllocateSpace implements ClientDriverExtensionAllocate +func (c *Connection) AllocateSpace(size int) error { + c.UpdateLastActivity() + // we don't have a path here so we check home dir and any virtual folders + // we return no error if there is space in any folder + folders := []string{"/"} + for _, v := range c.User.VirtualFolders { + // the space is checked for the parent folder + folders = append(folders, path.Join(v.VirtualPath, "fakefile.txt")) + } + for _, f := range folders { + quotaResult := c.HasSpace(false, f) + if quotaResult.HasSpace { + if quotaResult.QuotaSize == 0 { + // unlimited size is allowed + return nil + } + if quotaResult.GetRemainingSize() > int64(size) { + return nil + } + } + } + return common.ErrQuotaExceeded +} + +// ReadDir implements ClientDriverExtensionFilelist +func (c *Connection) ReadDir(name string) ([]os.FileInfo, error) { + c.UpdateLastActivity() + + p, err := c.Fs.ResolvePath(name) + if err != nil { + return nil, c.GetFsError(err) + } + return c.ListDir(p, name) +} + +// GetHandle implements ClientDriverExtentionFileTransfer +func (c *Connection) GetHandle(name string, flags int) (ftpserver.FileTransfer, error) { + c.UpdateLastActivity() + + p, err := c.Fs.ResolvePath(name) + if err != nil { + return nil, c.GetFsError(err) + } + if flags&os.O_WRONLY != 0 { + return c.uploadFile(p, name, flags) + } + return c.downloadFile(p, name) +} + +func (c *Connection) downloadFile(fsPath, ftpPath string) (ftpserver.FileTransfer, error) { + if !c.User.HasPerm(dataprovider.PermDownload, path.Dir(ftpPath)) { + return nil, c.GetPermissionDeniedError() + } + + if !c.User.IsFileAllowed(ftpPath) { + c.Log(logger.LevelWarn, "reading file %#v is not allowed", ftpPath) + return nil, c.GetPermissionDeniedError() + } + + file, r, cancelFn, err := c.Fs.Open(fsPath) + if err != nil { + c.Log(logger.LevelWarn, "could not open file %#v for reading: %+v", fsPath, err) + return nil, c.GetFsError(err) + } + + baseTransfer := common.NewBaseTransfer(file, c.BaseConnection, cancelFn, fsPath, ftpPath, common.TransferDownload, + 0, 0, false) + t := newTransfer(baseTransfer, nil, r, 0) + + return t, nil +} + +func (c *Connection) uploadFile(fsPath, ftpPath string, flags int) (ftpserver.FileTransfer, error) { + if !c.User.IsFileAllowed(ftpPath) { + c.Log(logger.LevelWarn, "writing file %#v is not allowed", ftpPath) + return nil, c.GetPermissionDeniedError() + } + + filePath := fsPath + if common.Config.IsAtomicUploadEnabled() && c.Fs.IsAtomicUploadSupported() { + filePath = c.Fs.GetAtomicUploadPath(fsPath) + } + + stat, statErr := c.Fs.Lstat(fsPath) + if (statErr == nil && stat.Mode()&os.ModeSymlink == os.ModeSymlink) || c.Fs.IsNotExist(statErr) { + if !c.User.HasPerm(dataprovider.PermUpload, path.Dir(ftpPath)) { + return nil, c.GetPermissionDeniedError() + } + return c.handleFTPUploadToNewFile(fsPath, filePath, ftpPath) + } + + if statErr != nil { + c.Log(logger.LevelError, "error performing file stat %#v: %+v", fsPath, statErr) + return nil, c.GetFsError(statErr) + } + + // This happen if we upload a file that has the same name of an existing directory + if stat.IsDir() { + c.Log(logger.LevelWarn, "attempted to open a directory for writing to: %#v", fsPath) + return nil, c.GetOpUnsupportedError() + } + + if !c.User.HasPerm(dataprovider.PermOverwrite, path.Dir(ftpPath)) { + return nil, c.GetPermissionDeniedError() + } + + return c.handleFTPUploadToExistingFile(flags, fsPath, filePath, stat.Size(), ftpPath) +} + +func (c *Connection) handleFTPUploadToNewFile(resolvedPath, filePath, requestPath string) (ftpserver.FileTransfer, error) { + quotaResult := c.HasSpace(true, requestPath) + if !quotaResult.HasSpace { + c.Log(logger.LevelInfo, "denying file write due to quota limits") + return nil, common.ErrQuotaExceeded + } + file, w, cancelFn, err := c.Fs.Create(filePath, 0) + if err != nil { + c.Log(logger.LevelWarn, "error creating file %#v: %+v", resolvedPath, err) + return nil, c.GetFsError(err) + } + + vfs.SetPathPermissions(c.Fs, filePath, c.User.GetUID(), c.User.GetGID()) + + baseTransfer := common.NewBaseTransfer(file, c.BaseConnection, cancelFn, resolvedPath, requestPath, + common.TransferUpload, 0, 0, true) + t := newTransfer(baseTransfer, w, nil, quotaResult.GetRemainingSize()) + + return t, nil +} + +func (c *Connection) handleFTPUploadToExistingFile(flags int, resolvedPath, filePath string, fileSize int64, + requestPath string) (ftpserver.FileTransfer, error) { + var err error + quotaResult := c.HasSpace(false, requestPath) + if !quotaResult.HasSpace { + c.Log(logger.LevelInfo, "denying file write due to quota limits") + return nil, common.ErrQuotaExceeded + } + minWriteOffset := int64(0) + + if flags&os.O_APPEND != 0 && flags&os.O_TRUNC == 0 && !c.Fs.IsUploadResumeSupported() { + c.Log(logger.LevelInfo, "upload resume requested for path: %#v but not supported in fs implementation", resolvedPath) + return nil, c.GetOpUnsupportedError() + } + + if common.Config.IsAtomicUploadEnabled() && c.Fs.IsAtomicUploadSupported() { + err = c.Fs.Rename(resolvedPath, filePath) + if err != nil { + c.Log(logger.LevelWarn, "error renaming existing file for atomic upload, source: %#v, dest: %#v, err: %+v", + resolvedPath, filePath, err) + return nil, c.GetFsError(err) + } + } + + file, w, cancelFn, err := c.Fs.Create(filePath, flags) + if err != nil { + c.Log(logger.LevelWarn, "error opening existing file, flags: %v, source: %#v, err: %+v", flags, filePath, err) + return nil, c.GetFsError(err) + } + + initialSize := int64(0) + // if there is a size limit remaining size cannot be 0 here, since quotaResult.HasSpace + // will return false in this case and we deny the upload before + maxWriteSize := quotaResult.GetRemainingSize() + if flags&os.O_APPEND != 0 && flags&os.O_TRUNC == 0 { + c.Log(logger.LevelDebug, "upload resume requested, file path: %#v initial size: %v", filePath, fileSize) + minWriteOffset = fileSize + } else { + if vfs.IsLocalOsFs(c.Fs) { + vfolder, err := c.User.GetVirtualFolderForPath(path.Dir(requestPath)) + if err == nil { + dataprovider.UpdateVirtualFolderQuota(vfolder.BaseVirtualFolder, 0, -fileSize, false) //nolint:errcheck + if vfolder.IsIncludedInUserQuota() { + dataprovider.UpdateUserQuota(c.User, 0, -fileSize, false) //nolint:errcheck + } + } else { + dataprovider.UpdateUserQuota(c.User, 0, -fileSize, false) //nolint:errcheck + } + } else { + initialSize = fileSize + } + if maxWriteSize > 0 { + maxWriteSize += fileSize + } + } + + vfs.SetPathPermissions(c.Fs, filePath, c.User.GetUID(), c.User.GetGID()) + + baseTransfer := common.NewBaseTransfer(file, c.BaseConnection, cancelFn, resolvedPath, requestPath, + common.TransferUpload, minWriteOffset, initialSize, false) + t := newTransfer(baseTransfer, w, nil, maxWriteSize) + + return t, nil +} diff --git a/ftpd/internal_test.go b/ftpd/internal_test.go new file mode 100644 index 00000000..c37e6dd6 --- /dev/null +++ b/ftpd/internal_test.go @@ -0,0 +1,441 @@ +package ftpd + +import ( + "fmt" + "io/ioutil" + "net" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/eikenb/pipeat" + "github.com/stretchr/testify/assert" + + "github.com/drakkan/sftpgo/common" + "github.com/drakkan/sftpgo/dataprovider" + "github.com/drakkan/sftpgo/vfs" +) + +const ( + configDir = ".." +) + +type mockFTPClientContext struct { +} + +func (cc mockFTPClientContext) Path() string { + return "" +} + +func (cc mockFTPClientContext) SetDebug(debug bool) {} + +func (cc mockFTPClientContext) Debug() bool { + return false +} + +func (cc mockFTPClientContext) ID() uint32 { + return 1 +} + +func (cc mockFTPClientContext) RemoteAddr() net.Addr { + return &net.IPAddr{IP: []byte("127.0.0.1")} +} + +func (cc mockFTPClientContext) LocalAddr() net.Addr { + return &net.IPAddr{IP: []byte("127.0.0.1")} +} + +func (cc mockFTPClientContext) GetClientVersion() string { + return "mock version" +} + +func (cc mockFTPClientContext) Close(code int, message string) error { + return nil +} + +// MockOsFs mockable OsFs +type MockOsFs struct { + vfs.Fs + err error + statErr error + isAtomicUploadSupported bool +} + +// Name returns the name for the Fs implementation +func (fs MockOsFs) Name() string { + return "mockOsFs" +} + +// IsUploadResumeSupported returns true if upload resume is supported +func (MockOsFs) IsUploadResumeSupported() bool { + return false +} + +// IsAtomicUploadSupported returns true if atomic upload is supported +func (fs MockOsFs) IsAtomicUploadSupported() bool { + return fs.isAtomicUploadSupported +} + +// Stat returns a FileInfo describing the named file +func (fs MockOsFs) Stat(name string) (os.FileInfo, error) { + if fs.statErr != nil { + return nil, fs.statErr + } + return os.Stat(name) +} + +// Lstat returns a FileInfo describing the named file +func (fs MockOsFs) Lstat(name string) (os.FileInfo, error) { + if fs.statErr != nil { + return nil, fs.statErr + } + return os.Lstat(name) +} + +// Remove removes the named file or (empty) directory. +func (fs MockOsFs) Remove(name string, isDir bool) error { + if fs.err != nil { + return fs.err + } + return os.Remove(name) +} + +// Rename renames (moves) source to target +func (fs MockOsFs) Rename(source, target string) error { + if fs.err != nil { + return fs.err + } + return os.Rename(source, target) +} + +func newMockOsFs(err, statErr error, atomicUpload bool, connectionID, rootDir string) vfs.Fs { + return &MockOsFs{ + Fs: vfs.NewOsFs(connectionID, rootDir, nil), + err: err, + statErr: statErr, + isAtomicUploadSupported: atomicUpload, + } +} + +func TestInitialization(t *testing.T) { + c := &Configuration{ + BindPort: 2121, + CertificateFile: "acert", + CertificateKeyFile: "akey", + } + err := c.Initialize(configDir) + assert.Error(t, err) + c.CertificateFile = "" + c.CertificateKeyFile = "" + c.BannerFile = "afile" + server, err := NewServer(c, configDir) + if assert.NoError(t, err) { + assert.Equal(t, "", server.initialMsg) + _, err = server.GetTLSConfig() + assert.Error(t, err) + } + err = ReloadTLSCertificate() + assert.NoError(t, err) +} + +func TestServerGetSettings(t *testing.T) { + oldConfig := common.Config + c := &Configuration{ + BindPort: 2121, + PassivePortRange: PortRange{ + Start: 10000, + End: 11000, + }, + } + server, err := NewServer(c, configDir) + assert.NoError(t, err) + settings, err := server.GetSettings() + assert.NoError(t, err) + assert.Equal(t, 10000, settings.PassiveTransferPortRange.Start) + assert.Equal(t, 11000, settings.PassiveTransferPortRange.End) + + common.Config.ProxyProtocol = 1 + common.Config.ProxyAllowed = []string{"invalid"} + _, err = server.GetSettings() + assert.Error(t, err) + server.config.BindPort = 8021 + _, err = server.GetSettings() + assert.Error(t, err) + + common.Config = oldConfig +} + +func TestUserInvalidParams(t *testing.T) { + u := dataprovider.User{ + HomeDir: "invalid", + } + c := &Configuration{ + BindPort: 2121, + PassivePortRange: PortRange{ + Start: 10000, + End: 11000, + }, + } + server, err := NewServer(c, configDir) + assert.NoError(t, err) + _, err = server.validateUser(u, mockFTPClientContext{}) + assert.Error(t, err) + + u.Username = "a" + u.HomeDir = filepath.Clean(os.TempDir()) + subDir := "subdir" + mappedPath1 := filepath.Join(os.TempDir(), "vdir1") + vdirPath1 := "/vdir1" + mappedPath2 := filepath.Join(os.TempDir(), "vdir1", subDir) + vdirPath2 := "/vdir2" + u.VirtualFolders = append(u.VirtualFolders, vfs.VirtualFolder{ + BaseVirtualFolder: vfs.BaseVirtualFolder{ + MappedPath: mappedPath1, + }, + VirtualPath: vdirPath1, + }) + u.VirtualFolders = append(u.VirtualFolders, vfs.VirtualFolder{ + BaseVirtualFolder: vfs.BaseVirtualFolder{ + MappedPath: mappedPath2, + }, + VirtualPath: vdirPath2, + }) + _, err = server.validateUser(u, mockFTPClientContext{}) + assert.Error(t, err) + u.VirtualFolders = nil + _, err = server.validateUser(u, mockFTPClientContext{}) + assert.Error(t, err) +} + +func TestClientVersion(t *testing.T) { + mockCC := mockFTPClientContext{} + connID := fmt.Sprintf("%v", mockCC.ID()) + user := dataprovider.User{} + connection := &Connection{ + BaseConnection: common.NewBaseConnection(connID, common.ProtocolFTP, user, nil), + clientContext: mockCC, + } + common.Connections.Add(connection) + stats := common.Connections.GetStats() + if assert.Len(t, stats, 1) { + assert.Equal(t, "mock version", stats[0].ClientVersion) + common.Connections.Remove(connection.GetID()) + } + assert.Len(t, common.Connections.GetStats(), 0) +} + +func TestDriverMethodsNotImplemented(t *testing.T) { + mockCC := mockFTPClientContext{} + connID := fmt.Sprintf("%v", mockCC.ID()) + user := dataprovider.User{} + connection := &Connection{ + BaseConnection: common.NewBaseConnection(connID, common.ProtocolFTP, user, nil), + clientContext: mockCC, + } + _, err := connection.Create("") + assert.EqualError(t, err, errNotImplemented.Error()) + err = connection.MkdirAll("", os.ModePerm) + assert.EqualError(t, err, errNotImplemented.Error()) + _, err = connection.Open("") + assert.EqualError(t, err, errNotImplemented.Error()) + _, err = connection.OpenFile("", 0, os.ModePerm) + assert.EqualError(t, err, errNotImplemented.Error()) + err = connection.RemoveAll("") + assert.EqualError(t, err, errNotImplemented.Error()) + assert.Equal(t, connection.GetID(), connection.Name()) +} + +func TestResolvePathErrors(t *testing.T) { + user := dataprovider.User{ + HomeDir: "invalid", + } + user.Permissions = make(map[string][]string) + user.Permissions["/"] = []string{dataprovider.PermAny} + mockCC := mockFTPClientContext{} + connID := fmt.Sprintf("%v", mockCC.ID()) + fs := vfs.NewOsFs(connID, user.HomeDir, nil) + connection := &Connection{ + BaseConnection: common.NewBaseConnection(connID, common.ProtocolFTP, user, fs), + clientContext: mockCC, + } + err := connection.Mkdir("", os.ModePerm) + if assert.Error(t, err) { + assert.EqualError(t, err, common.ErrGenericFailure.Error()) + } + err = connection.Remove("") + if assert.Error(t, err) { + assert.EqualError(t, err, common.ErrGenericFailure.Error()) + } + err = connection.Rename("", "") + if assert.Error(t, err) { + assert.EqualError(t, err, common.ErrGenericFailure.Error()) + } + _, err = connection.Stat("") + if assert.Error(t, err) { + assert.EqualError(t, err, common.ErrGenericFailure.Error()) + } + err = connection.Chmod("", os.ModePerm) + if assert.Error(t, err) { + assert.EqualError(t, err, common.ErrGenericFailure.Error()) + } + err = connection.Chtimes("", time.Now(), time.Now()) + if assert.Error(t, err) { + assert.EqualError(t, err, common.ErrGenericFailure.Error()) + } + _, err = connection.ReadDir("") + if assert.Error(t, err) { + assert.EqualError(t, err, common.ErrGenericFailure.Error()) + } + _, err = connection.GetHandle("", 0) + if assert.Error(t, err) { + assert.EqualError(t, err, common.ErrGenericFailure.Error()) + } +} + +func TestUploadFileStatError(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("this test is not available on Windows") + } + user := dataprovider.User{ + Username: "user", + HomeDir: filepath.Clean(os.TempDir()), + } + user.Permissions = make(map[string][]string) + user.Permissions["/"] = []string{dataprovider.PermAny} + mockCC := mockFTPClientContext{} + connID := fmt.Sprintf("%v", mockCC.ID()) + fs := vfs.NewOsFs(connID, user.HomeDir, nil) + connection := &Connection{ + BaseConnection: common.NewBaseConnection(connID, common.ProtocolFTP, user, fs), + clientContext: mockCC, + } + testFile := filepath.Join(user.HomeDir, "test", "testfile") + err := os.MkdirAll(filepath.Dir(testFile), os.ModePerm) + assert.NoError(t, err) + err = ioutil.WriteFile(testFile, []byte("data"), os.ModePerm) + assert.NoError(t, err) + err = os.Chmod(filepath.Dir(testFile), 0001) + assert.NoError(t, err) + _, err = connection.uploadFile(testFile, "test", 0) + assert.Error(t, err) + err = os.Chmod(filepath.Dir(testFile), os.ModePerm) + assert.NoError(t, err) + err = os.RemoveAll(filepath.Dir(testFile)) + assert.NoError(t, err) +} + +func TestUploadOverwriteErrors(t *testing.T) { + user := dataprovider.User{ + Username: "user", + HomeDir: filepath.Clean(os.TempDir()), + } + user.Permissions = make(map[string][]string) + user.Permissions["/"] = []string{dataprovider.PermAny} + mockCC := mockFTPClientContext{} + connID := fmt.Sprintf("%v", mockCC.ID()) + fs := newMockOsFs(nil, nil, false, connID, user.GetHomeDir()) + connection := &Connection{ + BaseConnection: common.NewBaseConnection(connID, common.ProtocolFTP, user, fs), + clientContext: mockCC, + } + flags := 0 + flags |= os.O_APPEND + _, err := connection.handleFTPUploadToExistingFile(flags, "", "", 0, "") + if assert.Error(t, err) { + assert.EqualError(t, err, common.ErrOpUnsupported.Error()) + } + + f, err := ioutil.TempFile("", "temp") + assert.NoError(t, err) + err = f.Close() + assert.NoError(t, err) + flags = 0 + flags |= os.O_CREATE + flags |= os.O_TRUNC + tr, err := connection.handleFTPUploadToExistingFile(flags, f.Name(), f.Name(), 123, f.Name()) + if assert.NoError(t, err) { + transfer := tr.(*transfer) + transfers := connection.GetTransfers() + if assert.Equal(t, 1, len(transfers)) { + assert.Equal(t, transfers[0].ID, transfer.GetID()) + assert.Equal(t, int64(123), transfer.InitialSize) + err = transfer.Close() + assert.NoError(t, err) + assert.Equal(t, 0, len(connection.GetTransfers())) + } + } + err = os.Remove(f.Name()) + assert.NoError(t, err) + + _, err = connection.handleFTPUploadToExistingFile(0, filepath.Join(os.TempDir(), "sub", "file"), + filepath.Join(os.TempDir(), "sub", "file1"), 0, "/sub/file1") + assert.Error(t, err) + connection.Fs = vfs.NewOsFs(connID, user.GetHomeDir(), nil) + _, err = connection.handleFTPUploadToExistingFile(0, "missing1", "missing2", 0, "missing") + assert.Error(t, err) +} + +func TestTransferErrors(t *testing.T) { + testfile := "testfile" + file, err := os.Create(testfile) + assert.NoError(t, err) + user := dataprovider.User{ + Username: "user", + HomeDir: filepath.Clean(os.TempDir()), + } + user.Permissions = make(map[string][]string) + user.Permissions["/"] = []string{dataprovider.PermAny} + mockCC := mockFTPClientContext{} + connID := fmt.Sprintf("%v", mockCC.ID()) + fs := newMockOsFs(nil, nil, false, connID, user.GetHomeDir()) + connection := &Connection{ + BaseConnection: common.NewBaseConnection(connID, common.ProtocolFTP, user, fs), + clientContext: mockCC, + } + baseTransfer := common.NewBaseTransfer(file, connection.BaseConnection, nil, file.Name(), testfile, common.TransferDownload, + 0, 0, false) + tr := newTransfer(baseTransfer, nil, nil, 0) + err = tr.Close() + assert.NoError(t, err) + buf := make([]byte, 64) + _, err = tr.Read(buf) + assert.Error(t, err) + err = tr.Close() + if assert.Error(t, err) { + assert.EqualError(t, err, common.ErrTransferClosed.Error()) + } + assert.Len(t, connection.GetTransfers(), 0) + + r, _, err := pipeat.Pipe() + assert.NoError(t, err) + baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testfile, testfile, + common.TransferUpload, 0, 0, false) + tr = newTransfer(baseTransfer, nil, r, 0) + err = tr.closeIO() + assert.NoError(t, err) + + r, w, err := pipeat.Pipe() + assert.NoError(t, err) + pipeWriter := vfs.NewPipeWriter(w) + baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testfile, testfile, + common.TransferUpload, 0, 0, false) + tr = newTransfer(baseTransfer, pipeWriter, nil, 0) + _, err = tr.Seek(1, 0) + if assert.Error(t, err) { + assert.EqualError(t, err, common.ErrOpUnsupported.Error()) + } + + err = r.Close() + assert.NoError(t, err) + errFake := fmt.Errorf("fake upload error") + go func() { + time.Sleep(100 * time.Millisecond) + pipeWriter.Done(errFake) + }() + err = tr.closeIO() + assert.EqualError(t, err, errFake.Error()) + err = os.Remove(testfile) + assert.NoError(t, err) +} diff --git a/ftpd/server.go b/ftpd/server.go new file mode 100644 index 00000000..ed99debc --- /dev/null +++ b/ftpd/server.go @@ -0,0 +1,190 @@ +package ftpd + +import ( + "crypto/tls" + "errors" + "fmt" + "io/ioutil" + "net" + "path/filepath" + + ftpserver "github.com/fclairamb/ftpserverlib" + + "github.com/drakkan/sftpgo/common" + "github.com/drakkan/sftpgo/dataprovider" + "github.com/drakkan/sftpgo/logger" + "github.com/drakkan/sftpgo/metrics" + "github.com/drakkan/sftpgo/utils" +) + +// Server implements the ftpserverlib MainDriver interface +type Server struct { + config *Configuration + certMgr *common.CertManager + initialMsg string +} + +// NewServer returns a new FTP server driver +func NewServer(config *Configuration, configDir string) (*Server, error) { + var err error + server := &Server{ + config: config, + certMgr: nil, + initialMsg: config.Banner, + } + certificateFile := getConfigPath(config.CertificateFile, configDir) + certificateKeyFile := getConfigPath(config.CertificateKeyFile, configDir) + if len(certificateFile) > 0 && len(certificateKeyFile) > 0 { + server.certMgr, err = common.NewCertManager(certificateFile, certificateKeyFile, logSender) + if err != nil { + return server, err + } + } + if len(config.BannerFile) > 0 { + bannerFilePath := config.BannerFile + if !filepath.IsAbs(bannerFilePath) { + bannerFilePath = filepath.Join(configDir, bannerFilePath) + } + bannerContent, err := ioutil.ReadFile(bannerFilePath) + if err == nil { + server.initialMsg = string(bannerContent) + } else { + logger.WarnToConsole("unable to read FTPD banner file: %v", err) + logger.Warn(logSender, "", "unable to read banner file: %v", err) + } + } + return server, err +} + +// GetSettings returns FTP server settings +func (s *Server) GetSettings() (*ftpserver.Settings, error) { + var portRange *ftpserver.PortRange = nil + if s.config.PassivePortRange.Start > 0 && s.config.PassivePortRange.End > s.config.PassivePortRange.Start { + portRange = &ftpserver.PortRange{ + Start: s.config.PassivePortRange.Start, + End: s.config.PassivePortRange.End, + } + } + var ftpListener net.Listener + if common.Config.ProxyProtocol > 0 { + listener, err := net.Listen("tcp", fmt.Sprintf("%s:%d", s.config.BindAddress, s.config.BindPort)) + if err != nil { + logger.Warn(logSender, "", "error starting listener on address %s:%d: %v", s.config.BindAddress, s.config.BindPort, err) + return nil, err + } + ftpListener, err = common.Config.GetProxyListener(listener) + if err != nil { + logger.Warn(logSender, "", "error enabling proxy listener: %v", err) + return nil, err + } + } + + return &ftpserver.Settings{ + Listener: ftpListener, + ListenAddr: fmt.Sprintf("%s:%d", s.config.BindAddress, s.config.BindPort), + PublicHost: s.config.ForcePassiveIP, + PassiveTransferPortRange: portRange, + ActiveTransferPortNon20: s.config.ActiveTransfersPortNon20, + IdleTimeout: -1, + ConnectionTimeout: 30, + }, nil +} + +// ClientConnected is called to send the very first welcome message +func (s *Server) ClientConnected(cc ftpserver.ClientContext) (string, error) { + connID := fmt.Sprintf("%v", cc.ID()) + user := dataprovider.User{} + connection := &Connection{ + BaseConnection: common.NewBaseConnection(connID, common.ProtocolFTP, user, nil), + clientContext: cc, + } + common.Connections.Add(connection) + return s.initialMsg, nil +} + +// ClientDisconnected is called when the user disconnects, even if he never authenticated +func (s *Server) ClientDisconnected(cc ftpserver.ClientContext) { + connID := fmt.Sprintf("%v_%v", common.ProtocolFTP, cc.ID()) + common.Connections.Remove(connID) +} + +// AuthUser authenticates the user and selects an handling driver +func (s *Server) AuthUser(cc ftpserver.ClientContext, username, password string) (ftpserver.ClientDriver, error) { + remoteAddr := cc.RemoteAddr().String() + user, err := dataprovider.CheckUserAndPass(username, password) + if err != nil { + updateLoginMetrics(username, remoteAddr, dataprovider.FTPLoginMethodPassword, err) + return nil, err + } + + connection, err := s.validateUser(user, cc) + + defer updateLoginMetrics(username, remoteAddr, dataprovider.FTPLoginMethodPassword, err) + + if err != nil { + return nil, err + } + connection.Fs.CheckRootPath(connection.GetUsername(), user.GetUID(), user.GetGID()) + connection.Log(logger.LevelInfo, "User id: %d, logged in with FTP, username: %#v, home_dir: %#v remote addr: %#v", + user.ID, user.Username, user.HomeDir, remoteAddr) + dataprovider.UpdateLastLogin(user) //nolint:errcheck + return connection, nil +} + +// GetTLSConfig returns a TLS Certificate to use +func (s *Server) GetTLSConfig() (*tls.Config, error) { + if s.certMgr != nil { + return &tls.Config{ + GetCertificate: s.certMgr.GetCertificateFunc(), + }, nil + } + return nil, errors.New("no TLS certificate configured") +} + +func (s *Server) validateUser(user dataprovider.User, cc ftpserver.ClientContext) (*Connection, error) { + connectionID := fmt.Sprintf("%v_%v", common.ProtocolFTP, cc.ID()) + if !filepath.IsAbs(user.HomeDir) { + logger.Warn(logSender, connectionID, "user %#v has an invalid home dir: %#v. Home dir must be an absolute path, login not allowed", + user.Username, user.HomeDir) + return nil, fmt.Errorf("cannot login user with invalid home dir: %#v", user.HomeDir) + } + if user.MaxSessions > 0 { + activeSessions := common.Connections.GetActiveSessions(user.Username) + if activeSessions >= user.MaxSessions { + logger.Debug(logSender, connectionID, "authentication refused for user: %#v, too many open sessions: %v/%v", user.Username, + activeSessions, user.MaxSessions) + return nil, fmt.Errorf("too many open sessions: %v", activeSessions) + } + } + if dataprovider.GetQuotaTracking() > 0 && user.HasOverlappedMappedPaths() { + logger.Debug(logSender, connectionID, "cannot login user %#v, overlapping mapped folders are allowed only with quota tracking disabled", + user.Username) + return nil, errors.New("overlapping mapped folders are allowed only with quota tracking disabled") + } + remoteAddr := cc.RemoteAddr().String() + if !user.IsLoginFromAddrAllowed(remoteAddr) { + logger.Debug(logSender, connectionID, "cannot login user %#v, remote address is not allowed: %v", user.Username, remoteAddr) + return nil, fmt.Errorf("Login for user %#v is not allowed from this address: %v", user.Username, remoteAddr) + } + fs, err := user.GetFilesystem(connectionID) + if err != nil { + return nil, err + } + connection := &Connection{ + BaseConnection: common.NewBaseConnection(fmt.Sprintf("%v", cc.ID()), common.ProtocolFTP, user, fs), + clientContext: cc, + } + err = common.Connections.Swap(connection) + if err != nil { + return nil, errors.New("Internal authentication error") + } + return connection, nil +} + +func updateLoginMetrics(username, remoteAddress, method string, err error) { + metrics.AddLoginAttempt(method) + if err != nil { + logger.ConnectionFailedLog(username, utils.GetIPFromRemoteAddress(remoteAddress), method, err.Error()) + } + metrics.AddLoginResult(method, err) +} diff --git a/ftpd/transfer.go b/ftpd/transfer.go new file mode 100644 index 00000000..8c289cc1 --- /dev/null +++ b/ftpd/transfer.go @@ -0,0 +1,128 @@ +package ftpd + +import ( + "io" + "sync/atomic" + + "github.com/eikenb/pipeat" + + "github.com/drakkan/sftpgo/common" + "github.com/drakkan/sftpgo/vfs" +) + +// transfer contains the transfer details for an upload or a download. +// It implements the ftpserver.FileTransfer interface to handle files downloads and uploads +type transfer struct { + *common.BaseTransfer + writer io.WriteCloser + reader io.ReadCloser + isFinished bool + maxWriteSize int64 +} + +func newTransfer(baseTransfer *common.BaseTransfer, pipeWriter *vfs.PipeWriter, pipeReader *pipeat.PipeReaderAt, + maxWriteSize int64) *transfer { + var writer io.WriteCloser + var reader io.ReadCloser + if baseTransfer.File != nil { + writer = baseTransfer.File + reader = baseTransfer.File + } else if pipeWriter != nil { + writer = pipeWriter + } else if pipeReader != nil { + reader = pipeReader + } + return &transfer{ + BaseTransfer: baseTransfer, + writer: writer, + reader: reader, + isFinished: false, + maxWriteSize: maxWriteSize, + } +} + +// Read reads the contents to downloads. +func (t *transfer) Read(p []byte) (n int, err error) { + t.Connection.UpdateLastActivity() + var readed int + var e error + + readed, e = t.reader.Read(p) + atomic.AddInt64(&t.BytesSent, int64(readed)) + + if e != nil && e != io.EOF { + t.TransferError(e) + return readed, e + } + t.HandleThrottle() + return readed, e +} + +// Write writes the uploaded contents. +func (t *transfer) Write(p []byte) (n int, err error) { + t.Connection.UpdateLastActivity() + var written int + var e error + + written, e = t.writer.Write(p) + atomic.AddInt64(&t.BytesReceived, int64(written)) + + if t.maxWriteSize > 0 && e == nil && atomic.LoadInt64(&t.BytesReceived) > t.maxWriteSize { + e = common.ErrQuotaExceeded + } + if e != nil { + t.TransferError(e) + return written, e + } + t.HandleThrottle() + return written, e +} + +// Seek sets the offset to resume an upload or a download +func (t *transfer) Seek(offset int64, whence int) (int64, error) { + if t.File != nil { + return t.File.Seek(offset, whence) + } + return 0, common.ErrOpUnsupported +} + +// Close it is called when the transfer is completed. +func (t *transfer) Close() error { + if err := t.setFinished(); err != nil { + return err + } + err := t.closeIO() + errBaseClose := t.BaseTransfer.Close() + if errBaseClose != nil { + err = errBaseClose + } + return t.Connection.GetFsError(err) +} + +func (t *transfer) closeIO() error { + var err error + if t.File != nil { + err = t.File.Close() + } else if t.writer != nil { + err = t.writer.Close() + t.Lock() + // we set ErrTransfer here so quota is not updated, in this case the uploads are atomic + if err != nil && t.ErrTransfer == nil { + t.ErrTransfer = err + } + t.Unlock() + } else if t.reader != nil { + err = t.reader.Close() + } + return err +} + +func (t *transfer) setFinished() error { + t.Lock() + defer t.Unlock() + if t.isFinished { + return common.ErrTransferClosed + } + t.isFinished = true + return nil +} diff --git a/go.mod b/go.mod index 68089fb1..986a3236 100644 --- a/go.mod +++ b/go.mod @@ -8,12 +8,14 @@ require ( github.com/alexedwards/argon2id v0.0.0-20200522061839-9369edc04b05 github.com/aws/aws-sdk-go v1.33.1 github.com/eikenb/pipeat v0.0.0-20200430215831-470df5986b6d + github.com/fclairamb/ftpserverlib v0.8.0 github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/go-chi/chi v4.1.2+incompatible github.com/go-chi/render v1.0.1 github.com/go-sql-driver/mysql v1.5.0 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/grandcat/zeroconf v1.0.0 + github.com/jlaffaye/ftp v0.0.0-20200720194710-13949d38913e github.com/lib/pq v1.7.0 github.com/mattn/go-sqlite3 v1.14.0 github.com/miekg/dns v1.1.29 // indirect @@ -26,7 +28,7 @@ require ( github.com/prometheus/client_golang v1.7.1 github.com/rs/xid v1.2.1 github.com/rs/zerolog v1.19.0 - github.com/spf13/afero v1.3.1 // indirect + github.com/spf13/afero v1.3.2 github.com/spf13/cast v1.3.1 // indirect github.com/spf13/cobra v1.0.0 github.com/spf13/jwalterweatherman v1.1.0 // indirect @@ -47,4 +49,8 @@ require ( gopkg.in/natefinch/lumberjack.v2 v2.0.0 ) -replace golang.org/x/crypto => github.com/drakkan/crypto v0.0.0-20200705203859-05ad140ecdbd +replace ( + github.com/fclairamb/ftpserverlib => github.com/drakkan/ftpserverlib v0.0.0-20200729185904-a61d63fc1db1 + github.com/jlaffaye/ftp => github.com/drakkan/ftp v0.0.0-20200727182237-9cca2b71337f + golang.org/x/crypto => github.com/drakkan/crypto v0.0.0-20200705203859-05ad140ecdbd +) diff --git a/go.sum b/go.sum index aeeb62b0..1db6d709 100644 --- a/go.sum +++ b/go.sum @@ -35,8 +35,13 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= +github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= +github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= +github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -44,18 +49,25 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alexedwards/argon2id v0.0.0-20200522061839-9369edc04b05 h1:votg1faEmwABhCeJ4tiBrvwk4BWftQGkEtFy5iuI7rU= github.com/alexedwards/argon2id v0.0.0-20200522061839-9369edc04b05/go.mod h1:GFtu6vaWaRJV5EvSFaVqgq/3Iq95xyYElBV/aupGzUo= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= +github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= +github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.33.1 h1:yz9XmNzPshz/lhfAZvLfMnIS9HPo8+boGRcWqDVX+T0= github.com/aws/aws-sdk-go v1.33.1/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -66,17 +78,24 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -84,13 +103,29 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/drakkan/crypto v0.0.0-20200705203859-05ad140ecdbd h1:7DQ8ayx6QylOxQrQ6oHdO+gk1cqtaINUekLrwD9gGNc= github.com/drakkan/crypto v0.0.0-20200705203859-05ad140ecdbd/go.mod h1:v3bhWOXGYda7H5d2s5t9XA6th3fxW3s0MQxU1R96G/w= +github.com/drakkan/ftp v0.0.0-20200727182237-9cca2b71337f h1:O58KACT8jWFlYJ3VSNFk+T+s7eX67LQwkC+FTRmCPa8= +github.com/drakkan/ftp v0.0.0-20200727182237-9cca2b71337f/go.mod h1:2lmrmq866uF2tnje75wQHzmPXhmSWUt7Gyx2vgK1RCU= +github.com/drakkan/ftpserverlib v0.0.0-20200728193209-1f77405f11b5 h1:pAThLlS4VfkAOPk/tex9/qVwsbb1CRT9YkBjsBVBFNI= +github.com/drakkan/ftpserverlib v0.0.0-20200728193209-1f77405f11b5/go.mod h1:Jwd+zOP3T0kwiCQcgjpu3VWtc7AI6Nu4UPN2HYqaniM= +github.com/drakkan/ftpserverlib v0.0.0-20200729180240-d4d48e974eed h1:mIaBQdWAOkh6Rt+O9sEo/gGJaC7DnPuxpqVpPZYz0z8= +github.com/drakkan/ftpserverlib v0.0.0-20200729180240-d4d48e974eed/go.mod h1:Jwd+zOP3T0kwiCQcgjpu3VWtc7AI6Nu4UPN2HYqaniM= +github.com/drakkan/ftpserverlib v0.0.0-20200729185904-a61d63fc1db1 h1:PkZAqIdsFJpfyRLmCOHZHsu3VUDgqClze0+hai6HBL8= +github.com/drakkan/ftpserverlib v0.0.0-20200729185904-a61d63fc1db1/go.mod h1:Jwd+zOP3T0kwiCQcgjpu3VWtc7AI6Nu4UPN2HYqaniM= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/eikenb/pipeat v0.0.0-20200430215831-470df5986b6d h1:8RvCRWer7TB2n+DKhW4uW15hRiqPmabSnSyYhju/Nuw= github.com/eikenb/pipeat v0.0.0-20200430215831-470df5986b6d/go.mod h1:+JPhBw5JdJrSF80r6xsSg1TYHjyAGxYs4X24VyUdMZU= +github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= +github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -104,14 +139,20 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -137,6 +178,7 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -157,19 +199,28 @@ github.com/google/pprof v0.0.0-20200507031123-427632fa3b1c/go.mod h1:ZgVRPoUq/hf github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grandcat/zeroconf v1.0.0 h1:uHhahLBKqwWBV6WZUDAT71044vwOTL+McW0mBJvo6kE= github.com/grandcat/zeroconf v1.0.0/go.mod h1:lTKmG1zh86XyCoUeIHSA4FJMBwCJiQmGfcP2PdzytEs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= @@ -180,6 +231,7 @@ github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerX github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -189,12 +241,18 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= @@ -211,11 +269,16 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.7.0 h1:h93mCPfUSkaul3Ka/VG8uZdmW1uMHDGxzu0NWHuJmHY= github.com/lib/pq v1.7.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= +github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= +github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA= github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= @@ -243,23 +306,51 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nathanaelle/password/v2 v2.0.1 h1:ItoCTdsuIWzilYmllQPa3DR3YoCXcpfxScWLqr8Ii2s= github.com/nathanaelle/password/v2 v2.0.1/go.mod h1:eaoT+ICQEPNtikBRIAatN8ThWwMhVG+r1jTw60BvPJk= +github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= +github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= +github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= +github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= +github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= +github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= +github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= +github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= +github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= +github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/otiai10/copy v1.2.0 h1:HvG945u96iNadPoG2/Ja2+AUJeW5YuFQMixq9yirC+k= github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw= github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= +github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.8.0 h1:Keo9qb7iRJs2voHvunFtuuYFsbWeOBh8/P9v/kVMFtw= github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs= +github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= +github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pires/go-proxyproto v0.1.3 h1:2XEuhsQluSNA5QIQkiUv8PfgZ51sNYIQkq/yFquiSQM= github.com/pires/go-proxyproto v0.1.3/go.mod h1:Odh9VFOZJCf9G8cLW5o435Xf1J95Jw9Gw5rnCjcwzAY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pkg/sftp v1.11.1-0.20200716191756-97b9df616e69 h1:qbrvNcVkxFdNuawO0LsSRRVMubCHOHdDTbb5EO/hiPE= github.com/pkg/sftp v1.11.1-0.20200716191756-97b9df616e69/go.mod h1:PIrgHN0+qgDmYTNiwryjoEqmXo9tv8aMwQ//Yg1xwIs= @@ -267,26 +358,35 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_golang v1.7.1 h1:NTGy1Ja9pByO+xAeH/qiWnLrKtr3hJPNjaVUwnjpdpA= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= @@ -296,7 +396,9 @@ github.com/rs/zerolog v1.19.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJ github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/secsy/goftp v0.0.0-20190720192957-f31499d7c79a/go.mod h1:MnkX001NG75g3p8bhFycnyIjeQoOjGL6CEIsdE/nKSY= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -304,24 +406,30 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.3.1 h1:GPTpEAuNr98px18yNQ66JllNil98wfRZ/5Ukny8FeQA= -github.com/spf13/afero v1.3.1/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= +github.com/spf13/afero v1.3.2 h1:GDarE4TJQI52kYSbSAmLiId1Elfj+xgSDqrUZxFhxlU= +github.com/spf13/afero v1.3.2/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -332,24 +440,36 @@ github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= +go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4 h1:LYy1Hy3MJdrCdMwwzxA/dRok4ejH+RwNGbuoD9fCjto= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -383,11 +503,13 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -398,6 +520,7 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -428,9 +551,11 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -440,10 +565,12 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -467,10 +594,12 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -488,6 +617,8 @@ golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -497,6 +628,7 @@ golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -517,6 +649,7 @@ golang.org/x/tools v0.0.0-20200702044944-0cc1aa72b347/go.mod h1:EkVYQZoAsY45+roY golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -533,6 +666,7 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0 h1:jMF5hhVfMkTZwHW1SDpKq5CkgWLXOb31Foaca9Zr3oM= google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= @@ -543,6 +677,7 @@ google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= @@ -567,11 +702,15 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200626011028-ee7919e894b5/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200702021140-07506425bd67 h1:4BC1C1i30F3MZeiIO6y6IIo4DxrtOwITK87bQl3lhFA= google.golang.org/genproto v0.0.0-20200702021140-07506425bd67/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= @@ -595,13 +734,19 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/dutchcoders/goftp.v1 v1.0.0-20170301105846-ed59a591ce14/go.mod h1:nzmlZQ+UqB5+55CRTV/dOaiK8OrPl6Co96Ob8lH4Wxw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -611,6 +756,7 @@ gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -621,3 +767,5 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= diff --git a/httpd/httpd_test.go b/httpd/httpd_test.go index cfb57e4f..4eaa1716 100644 --- a/httpd/httpd_test.go +++ b/httpd/httpd_test.go @@ -1274,14 +1274,14 @@ func TestProviderErrors(t *testing.T) { backupContent, err := json.Marshal(backupData) assert.NoError(t, err) backupFilePath := filepath.Join(backupsPath, "backup.json") - err = ioutil.WriteFile(backupFilePath, backupContent, 0666) + err = ioutil.WriteFile(backupFilePath, backupContent, os.ModePerm) assert.NoError(t, err) _, _, err = httpd.Loaddata(backupFilePath, "", "", http.StatusInternalServerError) assert.NoError(t, err) backupData.Folders = append(backupData.Folders, vfs.BaseVirtualFolder{MappedPath: os.TempDir()}) backupContent, err = json.Marshal(backupData) assert.NoError(t, err) - err = ioutil.WriteFile(backupFilePath, backupContent, 0666) + err = ioutil.WriteFile(backupFilePath, backupContent, os.ModePerm) assert.NoError(t, err) _, _, err = httpd.Loaddata(backupFilePath, "", "", http.StatusInternalServerError) assert.NoError(t, err) @@ -1422,7 +1422,7 @@ func TestLoaddata(t *testing.T) { backupContent, err := json.Marshal(backupData) assert.NoError(t, err) backupFilePath := filepath.Join(backupsPath, "backup.json") - err = ioutil.WriteFile(backupFilePath, backupContent, 0666) + err = ioutil.WriteFile(backupFilePath, backupContent, os.ModePerm) assert.NoError(t, err) _, _, err = httpd.Loaddata(backupFilePath, "a", "", http.StatusBadRequest) assert.NoError(t, err) @@ -1489,7 +1489,7 @@ func TestLoaddataMode(t *testing.T) { backupData.Users = append(backupData.Users, user) backupContent, _ := json.Marshal(backupData) backupFilePath := filepath.Join(backupsPath, "backup.json") - err := ioutil.WriteFile(backupFilePath, backupContent, 0666) + err := ioutil.WriteFile(backupFilePath, backupContent, os.ModePerm) assert.NoError(t, err) _, _, err = httpd.Loaddata(backupFilePath, "0", "0", http.StatusOK) assert.NoError(t, err) @@ -2821,7 +2821,7 @@ func createTestFile(path string, size int64) error { return err } } - return ioutil.WriteFile(path, content, 0666) + return ioutil.WriteFile(path, content, os.ModePerm) } func getMultipartFormData(values url.Values, fileFieldName, filePath string) (bytes.Buffer, string, error) { diff --git a/httpd/internal_test.go b/httpd/internal_test.go index 7f4af9da..fe80cca7 100644 --- a/httpd/internal_test.go +++ b/httpd/internal_test.go @@ -441,7 +441,7 @@ func TestBasicAuth(t *testing.T) { oldAuthPassword := authPassword authUserFile := filepath.Join(os.TempDir(), "http_users.txt") authUserData := []byte("test1:$2y$05$bcHSED7aO1cfLto6ZdDBOOKzlwftslVhtpIkRhAtSa4GuLmk5mola\n") - err := ioutil.WriteFile(authUserFile, authUserData, 0666) + err := ioutil.WriteFile(authUserFile, authUserData, os.ModePerm) assert.NoError(t, err) httpAuth, _ = newBasicAuthProvider(authUserFile) _, _, err = GetVersion(http.StatusUnauthorized) @@ -454,7 +454,7 @@ func TestBasicAuth(t *testing.T) { defer resp.Body.Close() assert.Equal(t, http.StatusUnauthorized, resp.StatusCode) authUserData = append(authUserData, []byte("test2:$apr1$gLnIkRIf$Xr/6aJfmIrihP4b2N2tcs/\n")...) - err = ioutil.WriteFile(authUserFile, authUserData, 0666) + err = ioutil.WriteFile(authUserFile, authUserData, os.ModePerm) assert.NoError(t, err) SetBaseURLAndCredentials(httpBaseURL, "test2", "password2") _, _, err = GetVersion(http.StatusOK) @@ -463,31 +463,31 @@ func TestBasicAuth(t *testing.T) { _, _, err = GetVersion(http.StatusOK) assert.Error(t, err) authUserData = append(authUserData, []byte("test3:$apr1$gLnIkRIf$Xr/6$aJfmIr$ihP4b2N2tcs/\n")...) - err = ioutil.WriteFile(authUserFile, authUserData, 0666) + err = ioutil.WriteFile(authUserFile, authUserData, os.ModePerm) assert.NoError(t, err) SetBaseURLAndCredentials(httpBaseURL, "test3", "wrong_password") _, _, err = GetVersion(http.StatusUnauthorized) assert.NoError(t, err) authUserData = append(authUserData, []byte("test4:$invalid$gLnIkRIf$Xr/6$aJfmIr$ihP4b2N2tcs/\n")...) - err = ioutil.WriteFile(authUserFile, authUserData, 0666) + err = ioutil.WriteFile(authUserFile, authUserData, os.ModePerm) assert.NoError(t, err) SetBaseURLAndCredentials(httpBaseURL, "test3", "password2") _, _, err = GetVersion(http.StatusUnauthorized) assert.NoError(t, err) if runtime.GOOS != "windows" { authUserData = append(authUserData, []byte("test5:$apr1$gLnIkRIf$Xr/6aJfmIrihP4b2N2tcs/\n")...) - err = ioutil.WriteFile(authUserFile, authUserData, 0666) + err = ioutil.WriteFile(authUserFile, authUserData, os.ModePerm) assert.NoError(t, err) err = os.Chmod(authUserFile, 0001) assert.NoError(t, err) SetBaseURLAndCredentials(httpBaseURL, "test5", "password2") _, _, err = GetVersion(http.StatusUnauthorized) assert.NoError(t, err) - err = os.Chmod(authUserFile, 0666) + err = os.Chmod(authUserFile, os.ModePerm) assert.NoError(t, err) } authUserData = append(authUserData, []byte("\"foo\"bar\"\r\n")...) - err = ioutil.WriteFile(authUserFile, authUserData, 0666) + err = ioutil.WriteFile(authUserFile, authUserData, os.ModePerm) assert.NoError(t, err) SetBaseURLAndCredentials(httpBaseURL, "test2", "password2") _, _, err = GetVersion(http.StatusUnauthorized) diff --git a/metrics/metrics.go b/metrics/metrics.go index c95dacca..b82b0147 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -36,40 +36,40 @@ var ( Help: "Total number of logged in users", }) - // totalUploads is the metric that reports the total number of successful SFTP/SCP uploads + // totalUploads is the metric that reports the total number of successful uploads totalUploads = promauto.NewCounter(prometheus.CounterOpts{ Name: "sftpgo_uploads_total", - Help: "The total number of successful SFTP/SCP uploads", + Help: "The total number of successful uploads", }) - // totalDownloads is the metric that reports the total number of successful SFTP/SCP downloads + // totalDownloads is the metric that reports the total number of successful downloads totalDownloads = promauto.NewCounter(prometheus.CounterOpts{ Name: "sftpgo_downloads_total", - Help: "The total number of successful SFTP/SCP downloads", + Help: "The total number of successful downloads", }) - // totalUploadErrors is the metric that reports the total number of SFTP/SCP upload errors + // totalUploadErrors is the metric that reports the total number of upload errors totalUploadErrors = promauto.NewCounter(prometheus.CounterOpts{ Name: "sftpgo_upload_errors_total", - Help: "The total number of SFTP/SCP upload errors", + Help: "The total number of upload errors", }) - // totalDownloadErrors is the metric that reports the total number of SFTP/SCP download errors + // totalDownloadErrors is the metric that reports the total number of download errors totalDownloadErrors = promauto.NewCounter(prometheus.CounterOpts{ Name: "sftpgo_download_errors_total", - Help: "The total number of SFTP/SCP download errors", + Help: "The total number of download errors", }) - // totalUploadSize is the metric that reports the total SFTP/SCP uploads size as bytes + // totalUploadSize is the metric that reports the total uploads size as bytes totalUploadSize = promauto.NewCounter(prometheus.CounterOpts{ Name: "sftpgo_upload_size", - Help: "The total SFTP/SCP upload size as bytes, partial uploads are included", + Help: "The total upload size as bytes, partial uploads are included", }) - // totalDownloadSize is the metric that reports the total SFTP/SCP downloads size as bytes + // totalDownloadSize is the metric that reports the total downloads size as bytes totalDownloadSize = promauto.NewCounter(prometheus.CounterOpts{ Name: "sftpgo_download_size", - Help: "The total SFTP/SCP download size as bytes, partial downloads are included", + Help: "The total download size as bytes, partial downloads are included", }) // totalSSHCommands is the metric that reports the total number of executed SSH commands @@ -90,6 +90,13 @@ var ( Help: "The total number of login attempts", }) + // totalNoAuthTryed is te metric that reports the total number of clients disconnected + // for inactivity before trying to login + totalNoAuthTryed = promauto.NewCounter(prometheus.CounterOpts{ + Name: "sftpgo_no_auth_total", + Help: "The total number of clients disconnected for inactivity before trying to login", + }) + // totalLoginOK is the metric that reports the total number of successful logins totalLoginOK = promauto.NewCounter(prometheus.CounterOpts{ Name: "sftpgo_login_ok_total", @@ -604,6 +611,12 @@ func AddLoginResult(authMethod string, err error) { } } +// AddNoAuthTryed increments the metric for clients disconnected +// for inactivity before trying to login +func AddNoAuthTryed() { + totalNoAuthTryed.Inc() +} + // HTTPRequestServed increments the metrics for HTTP requests func HTTPRequestServed(status int) { totalHTTPRequests.Inc() diff --git a/metrics/metrics_disabled.go b/metrics/metrics_disabled.go index 02e42357..3efdd524 100644 --- a/metrics/metrics_disabled.go +++ b/metrics/metrics_disabled.go @@ -60,6 +60,10 @@ func AddLoginAttempt(authMethod string) {} // AddLoginResult increments the metrics for login results func AddLoginResult(authMethod string, err error) {} +// AddNoAuthTryed increments the metric for clients disconnected +// for inactivity before trying to login +func AddNoAuthTryed() {} + // HTTPRequestServed increments the metrics for HTTP requests func HTTPRequestServed(status int) {} diff --git a/service/service.go b/service/service.go index 1b9f0f74..c7225430 100644 --- a/service/service.go +++ b/service/service.go @@ -77,12 +77,6 @@ func (s *Service) Start() error { return err } - httpConfig := config.GetHTTPConfig() - httpConfig.Initialize(s.ConfigDir) - - sftpdConf := config.GetSFTPDConfig() - httpdConf := config.GetHTTPDConfig() - if s.PortableMode == 1 { // create the user for portable mode err = dataprovider.AddUser(s.PortableUser) @@ -92,6 +86,19 @@ func (s *Service) Start() error { } } + httpConfig := config.GetHTTPConfig() + httpConfig.Initialize(s.ConfigDir) + + s.startServices() + + return nil +} + +func (s *Service) startServices() { + sftpdConf := config.GetSFTPDConfig() + ftpdConf := config.GetFTPDConfig() + httpdConf := config.GetHTTPDConfig() + go func() { logger.Debug(logSender, "", "initializing SFTP server with config %+v", sftpdConf) if err := sftpdConf.Initialize(s.ConfigDir); err != nil { @@ -117,7 +124,18 @@ func (s *Service) Start() error { logger.DebugToConsole("HTTP server not started, disabled in config file") } } - return nil + if ftpdConf.BindPort > 0 { + go func() { + if err := ftpdConf.Initialize(s.ConfigDir); err != nil { + logger.Error(logSender, "", "could not start FTP server: %v", err) + logger.ErrorToConsole("could not start FTP server: %v", err) + s.Error = err + } + s.Shutdown <- true + }() + } else { + logger.Debug(logSender, "", "FTP server not started, disabled in config file") + } } // Wait blocks until the service exits diff --git a/service/service_portable.go b/service/service_portable.go index ab85901d..b7e07e59 100644 --- a/service/service_portable.go +++ b/service/service_portable.go @@ -23,7 +23,8 @@ import ( ) // StartPortableMode starts the service in portable mode -func (s *Service) StartPortableMode(sftpdPort int, enabledSSHCommands []string, advertiseService, advertiseCredentials bool) error { +func (s *Service) StartPortableMode(sftpdPort, ftpPort int, enabledSSHCommands []string, advertiseService, advertiseCredentials bool, + ftpsCert, ftpsKey string) error { if s.PortableMode != 1 { return fmt.Errorf("service is not configured for portable mode") } @@ -67,14 +68,44 @@ func (s *Service) StartPortableMode(sftpdPort int, enabledSSHCommands []string, } config.SetSFTPDConfig(sftpdConf) + if ftpPort >= 0 { + ftpConf := config.GetFTPDConfig() + if ftpPort > 0 { + ftpConf.BindPort = ftpPort + } else { + ftpConf.BindPort = 49152 + rand.Intn(15000) + } + ftpConf.Banner = fmt.Sprintf("SFTPGo portable %v ready", version.Get().Version) + ftpConf.CertificateFile = ftpsCert + ftpConf.CertificateKeyFile = ftpsKey + config.SetFTPDConfig(ftpConf) + } + err = s.Start() if err != nil { return err } - var mDNSService *zeroconf.Server + + s.advertiseServices(advertiseService, advertiseCredentials) + + var ftpInfo string + if config.GetFTPDConfig().BindPort > 0 { + ftpInfo = fmt.Sprintf("FTP port: %v", config.GetFTPDConfig().BindPort) + } + logger.InfoToConsole("Portable mode ready, SFTP port: %v, user: %#v, password: %#v, public keys: %v, directory: %#v, "+ + "permissions: %+v, enabled ssh commands: %v file extensions filters: %+v %v", sftpdConf.BindPort, s.PortableUser.Username, + printablePassword, s.PortableUser.PublicKeys, s.getPortableDirToServe(), s.PortableUser.Permissions, + sftpdConf.EnabledSSHCommands, s.PortableUser.Filters.FileExtensions, ftpInfo) + return nil +} + +func (s *Service) advertiseServices(advertiseService, advertiseCredentials bool) { + var mDNSServiceSFTP *zeroconf.Server + var mDNSServiceFTP *zeroconf.Server + var err error if advertiseService { meta := []string{ - fmt.Sprintf("version=%v", version.GetAsString()), + fmt.Sprintf("version=%v", version.Get().Version), } if advertiseCredentials { logger.InfoToConsole("Advertising credentials via multicast DNS") @@ -85,7 +116,8 @@ func (s *Service) StartPortableMode(sftpdPort int, enabledSSHCommands []string, logger.InfoToConsole("Unable to advertise key based credentials via multicast DNS, we don't have the private key") } } - mDNSService, err = zeroconf.Register( + sftpdConf := config.GetSFTPDConfig() + mDNSServiceSFTP, err = zeroconf.Register( fmt.Sprintf("SFTPGo portable %v", sftpdConf.BindPort), // service instance name "_sftp-ssh._tcp", // service type and protocol "local.", // service domain @@ -94,28 +126,41 @@ func (s *Service) StartPortableMode(sftpdPort int, enabledSSHCommands []string, nil, // register on all network interfaces ) if err != nil { - mDNSService = nil + mDNSServiceSFTP = nil logger.WarnToConsole("Unable to advertise SFTP service via multicast DNS: %v", err) } else { logger.InfoToConsole("SFTP service advertised via multicast DNS") } + ftpdConf := config.GetFTPDConfig() + mDNSServiceFTP, err = zeroconf.Register( + fmt.Sprintf("SFTPGo portable %v", ftpdConf.BindPort), + "_ftp._tcp", + "local.", + ftpdConf.BindPort, + meta, + nil, + ) + if err != nil { + mDNSServiceFTP = nil + logger.WarnToConsole("Unable to advertise FTP service via multicast DNS: %v", err) + } else { + logger.InfoToConsole("FTP service advertised via multicast DNS") + } } sig := make(chan os.Signal, 1) signal.Notify(sig, os.Interrupt, syscall.SIGTERM) go func() { <-sig - if mDNSService != nil { - logger.InfoToConsole("unregistering multicast DNS service") - mDNSService.Shutdown() + if mDNSServiceSFTP != nil { + logger.InfoToConsole("unregistering multicast DNS SFTP service") + mDNSServiceSFTP.Shutdown() + } + if mDNSServiceFTP != nil { + logger.InfoToConsole("unregistering multicast DNS FTP service") + mDNSServiceFTP.Shutdown() } s.Stop() }() - - logger.InfoToConsole("Portable mode ready, SFTP port: %v, user: %#v, password: %#v, public keys: %v, directory: %#v, "+ - "permissions: %+v, enabled ssh commands: %v file extensions filters: %+v", sftpdConf.BindPort, s.PortableUser.Username, - printablePassword, s.PortableUser.PublicKeys, s.getPortableDirToServe(), s.PortableUser.Permissions, - sftpdConf.EnabledSSHCommands, s.PortableUser.Filters.FileExtensions) - return nil } func (s *Service) getPortableDirToServe() string { diff --git a/service/service_windows.go b/service/service_windows.go index 5b529f3f..ed5d51c0 100644 --- a/service/service_windows.go +++ b/service/service_windows.go @@ -12,6 +12,7 @@ import ( "golang.org/x/sys/windows/svc/mgr" "github.com/drakkan/sftpgo/dataprovider" + "github.com/drakkan/sftpgo/ftpd" "github.com/drakkan/sftpgo/httpd" "github.com/drakkan/sftpgo/logger" ) @@ -93,6 +94,10 @@ loop: if err != nil { logger.Warn(logSender, "", "error reloading TLS certificate: %v", err) } + err = ftpd.ReloadTLSCertificate() + if err != nil { + logger.Warn(logSender, "", "error reloading FTPD TLS certificate: %v", err) + } case rotateLogCmd: logger.Debug(logSender, "", "Received log file rotation request") err := logger.RotateLogFile() diff --git a/service/sighup_unix.go b/service/sighup_unix.go index 6f4dc8aa..d26efe6e 100644 --- a/service/sighup_unix.go +++ b/service/sighup_unix.go @@ -8,6 +8,7 @@ import ( "syscall" "github.com/drakkan/sftpgo/dataprovider" + "github.com/drakkan/sftpgo/ftpd" "github.com/drakkan/sftpgo/httpd" "github.com/drakkan/sftpgo/logger" ) @@ -26,6 +27,10 @@ func registerSigHup() { if err != nil { logger.Warn(logSender, "", "error reloading TLS certificate: %v", err) } + err = ftpd.ReloadTLSCertificate() + if err != nil { + logger.Warn(logSender, "", "error reloading FTPD TLS certificate: %v", err) + } } }() } diff --git a/sftpd/handler.go b/sftpd/handler.go index d63b3af6..7fe274cd 100644 --- a/sftpd/handler.go +++ b/sftpd/handler.go @@ -74,7 +74,7 @@ func (c *Connection) Fileread(request *sftp.Request) (io.ReaderAt, error) { baseTransfer := common.NewBaseTransfer(file, c.BaseConnection, cancelFn, p, request.Filepath, common.TransferDownload, 0, 0, false) - t := newTranfer(baseTransfer, nil, r, 0) + t := newTransfer(baseTransfer, nil, r, 0) return t, nil } @@ -164,14 +164,6 @@ func (c *Connection) Filecmd(request *sftp.Request) error { return sftp.ErrSSHFxOpUnsupported } - var fileLocation = p - if target != "" { - fileLocation = target - } - - // we return if we remove a file or a dir so source path or target path always exists here - vfs.SetPathPermissions(c.Fs, fileLocation, c.User.GetUID(), c.User.GetGID()) - return sftp.ErrSSHFxOk } @@ -276,7 +268,7 @@ func (c *Connection) handleSFTPUploadToNewFile(resolvedPath, filePath, requestPa baseTransfer := common.NewBaseTransfer(file, c.BaseConnection, cancelFn, resolvedPath, requestPath, common.TransferUpload, 0, 0, true) - t := newTranfer(baseTransfer, w, nil, quotaResult.GetRemainingSize()) + t := newTransfer(baseTransfer, w, nil, quotaResult.GetRemainingSize()) return t, nil } @@ -343,7 +335,7 @@ func (c *Connection) handleSFTPUploadToExistingFile(pflags sftp.FileOpenFlags, r baseTransfer := common.NewBaseTransfer(file, c.BaseConnection, cancelFn, resolvedPath, requestPath, common.TransferUpload, minWriteOffset, initialSize, false) - t := newTranfer(baseTransfer, w, nil, maxWriteSize) + t := newTransfer(baseTransfer, w, nil, maxWriteSize) return t, nil } diff --git a/sftpd/internal_test.go b/sftpd/internal_test.go index 0b87cf0c..88802324 100644 --- a/sftpd/internal_test.go +++ b/sftpd/internal_test.go @@ -159,7 +159,7 @@ func TestUploadResumeInvalidOffset(t *testing.T) { fs := vfs.NewOsFs("", os.TempDir(), nil) conn := common.NewBaseConnection("", common.ProtocolSFTP, user, fs) baseTransfer := common.NewBaseTransfer(file, conn, nil, file.Name(), testfile, common.TransferUpload, 10, 0, false) - transfer := newTranfer(baseTransfer, nil, nil, 0) + transfer := newTransfer(baseTransfer, nil, nil, 0) _, err = transfer.WriteAt([]byte("test"), 0) assert.Error(t, err, "upload with invalid offset must fail") if assert.Error(t, transfer.ErrTransfer) { @@ -187,7 +187,7 @@ func TestReadWriteErrors(t *testing.T) { fs := vfs.NewOsFs("", os.TempDir(), nil) conn := common.NewBaseConnection("", common.ProtocolSFTP, user, fs) baseTransfer := common.NewBaseTransfer(file, conn, nil, file.Name(), testfile, common.TransferDownload, 0, 0, false) - transfer := newTranfer(baseTransfer, nil, nil, 0) + transfer := newTransfer(baseTransfer, nil, nil, 0) err = file.Close() assert.NoError(t, err) _, err = transfer.WriteAt([]byte("test"), 0) @@ -201,7 +201,7 @@ func TestReadWriteErrors(t *testing.T) { r, _, err := pipeat.Pipe() assert.NoError(t, err) baseTransfer = common.NewBaseTransfer(nil, conn, nil, file.Name(), testfile, common.TransferDownload, 0, 0, false) - transfer = newTranfer(baseTransfer, nil, r, 0) + transfer = newTransfer(baseTransfer, nil, r, 0) err = transfer.closeIO() assert.NoError(t, err) _, err = transfer.ReadAt(buf, 0) @@ -211,7 +211,7 @@ func TestReadWriteErrors(t *testing.T) { assert.NoError(t, err) pipeWriter := vfs.NewPipeWriter(w) baseTransfer = common.NewBaseTransfer(nil, conn, nil, file.Name(), testfile, common.TransferDownload, 0, 0, false) - transfer = newTranfer(baseTransfer, pipeWriter, nil, 0) + transfer = newTransfer(baseTransfer, pipeWriter, nil, 0) err = r.Close() assert.NoError(t, err) @@ -243,7 +243,7 @@ func TestTransferCancelFn(t *testing.T) { fs := vfs.NewOsFs("", os.TempDir(), nil) conn := common.NewBaseConnection("", common.ProtocolSFTP, user, fs) baseTransfer := common.NewBaseTransfer(file, conn, cancelFn, file.Name(), testfile, common.TransferDownload, 0, 0, false) - transfer := newTranfer(baseTransfer, nil, nil, 0) + transfer := newTransfer(baseTransfer, nil, nil, 0) errFake := errors.New("fake error, this will trigger cancelFn") transfer.TransferError(errFake) @@ -273,7 +273,7 @@ func TestMockFsErrors(t *testing.T) { } testfile := filepath.Join(u.HomeDir, "testfile") request := sftp.NewRequest("Remove", testfile) - err := ioutil.WriteFile(testfile, []byte("test"), 0666) + err := ioutil.WriteFile(testfile, []byte("test"), os.ModePerm) assert.NoError(t, err) _, err = c.Filewrite(request) assert.EqualError(t, err, sftp.ErrSSHFxFailure.Error()) @@ -979,7 +979,7 @@ func TestSystemCommandErrors(t *testing.T) { sshCmd.connection.channel = &mockSSHChannel baseTransfer := common.NewBaseTransfer(nil, sshCmd.connection.BaseConnection, nil, "", "", common.TransferDownload, 0, 0, false) - transfer := newTranfer(baseTransfer, nil, nil, 0) + transfer := newTransfer(baseTransfer, nil, nil, 0) destBuff := make([]byte, 65535) dst := bytes.NewBuffer(destBuff) _, err = transfer.copyFromReaderToWriter(dst, sshCmd.connection.channel) @@ -1308,7 +1308,7 @@ func TestSCPErrorsMockFs(t *testing.T) { assert.EqualError(t, err, errFake.Error()) testfile := filepath.Join(u.HomeDir, "testfile") - err = ioutil.WriteFile(testfile, []byte("test"), 0666) + err = ioutil.WriteFile(testfile, []byte("test"), os.ModePerm) assert.NoError(t, err) stat, err := os.Stat(u.HomeDir) assert.NoError(t, err) @@ -1476,7 +1476,7 @@ func TestSCPDownloadFileData(t *testing.T) { args: []string{"-r", "-f", "/tmp"}, }, } - err := ioutil.WriteFile(testfile, []byte("test"), 0666) + err := ioutil.WriteFile(testfile, []byte("test"), os.ModePerm) assert.NoError(t, err) stat, err := os.Stat(testfile) assert.NoError(t, err) @@ -1531,7 +1531,7 @@ func TestSCPUploadFiledata(t *testing.T) { baseTransfer := common.NewBaseTransfer(file, scpCommand.connection.BaseConnection, nil, file.Name(), "/"+testfile, common.TransferDownload, 0, 0, true) - transfer := newTranfer(baseTransfer, nil, nil, 0) + transfer := newTransfer(baseTransfer, nil, nil, 0) err = scpCommand.getUploadFileData(2, transfer) assert.Error(t, err, "upload must fail, we send a fake write error message") @@ -1563,7 +1563,7 @@ func TestSCPUploadFiledata(t *testing.T) { file, err = os.Create(testfile) assert.NoError(t, err) baseTransfer.File = file - transfer = newTranfer(baseTransfer, nil, nil, 0) + transfer = newTransfer(baseTransfer, nil, nil, 0) transfer.Connection.AddTransfer(transfer) err = scpCommand.getUploadFileData(2, transfer) assert.Error(t, err, "upload must fail, we have not enough data to read") @@ -1615,7 +1615,7 @@ func TestUploadError(t *testing.T) { assert.NoError(t, err) baseTransfer := common.NewBaseTransfer(file, connection.BaseConnection, nil, testfile, testfile, common.TransferUpload, 0, 0, true) - transfer := newTranfer(baseTransfer, nil, nil, 0) + transfer := newTransfer(baseTransfer, nil, nil, 0) errFake := errors.New("fake error") transfer.TransferError(errFake) @@ -1678,7 +1678,7 @@ func TestLoadHostKeys(t *testing.T) { err := c.checkAndLoadHostKeys(configDir, serverConfig) assert.Error(t, err) testfile := filepath.Join(os.TempDir(), "invalidkey") - err = ioutil.WriteFile(testfile, []byte("some bytes"), 0666) + err = ioutil.WriteFile(testfile, []byte("some bytes"), os.ModePerm) assert.NoError(t, err) c.HostKeys = []string{testfile} err = c.checkAndLoadHostKeys(configDir, serverConfig) @@ -1726,7 +1726,7 @@ func TestCertCheckerInitErrors(t *testing.T) { err := c.initializeCertChecker("") assert.Error(t, err) testfile := filepath.Join(os.TempDir(), "invalidkey") - err = ioutil.WriteFile(testfile, []byte("some bytes"), 0666) + err = ioutil.WriteFile(testfile, []byte("some bytes"), os.ModePerm) assert.NoError(t, err) c.TrustedUserCAKeys = []string{testfile} err = c.initializeCertChecker("") diff --git a/sftpd/scp.go b/sftpd/scp.go index ef3675b5..b7c89b5d 100644 --- a/sftpd/scp.go +++ b/sftpd/scp.go @@ -30,7 +30,7 @@ type scpCommand struct { func (c *scpCommand) handle() error { common.Connections.Add(c.connection) - defer common.Connections.Remove(c.connection) + defer common.Connections.Remove(c.connection.GetID()) var err error destPath := c.getDestPath() @@ -226,7 +226,7 @@ func (c *scpCommand) handleUploadFile(resolvedPath, filePath string, sizeToRead baseTransfer := common.NewBaseTransfer(file, c.connection.BaseConnection, cancelFn, resolvedPath, requestPath, common.TransferUpload, 0, initialSize, isNewFile) - t := newTranfer(baseTransfer, w, nil, maxWriteSize) + t := newTransfer(baseTransfer, w, nil, maxWriteSize) return c.getUploadFileData(sizeToRead, t) } @@ -484,7 +484,7 @@ func (c *scpCommand) handleDownload(filePath string) error { baseTransfer := common.NewBaseTransfer(file, c.connection.BaseConnection, cancelFn, p, filePath, common.TransferDownload, 0, 0, false) - t := newTranfer(baseTransfer, nil, r, 0) + t := newTransfer(baseTransfer, nil, r, 0) err = c.sendDownloadFileData(p, stat, t) // we need to call Close anyway and return close error if any and diff --git a/sftpd/server.go b/sftpd/server.go index 78244acb..c5f9684d 100644 --- a/sftpd/server.go +++ b/sftpd/server.go @@ -223,7 +223,7 @@ func (c Configuration) configureLoginBanner(serverConfig *ssh.ServerConfig, conf return banner } } else { - logger.WarnToConsole("unable to read login banner file: %v", err) + logger.WarnToConsole("unable to read SFTPD login banner file: %v", err) logger.Warn(logSender, "", "unable to read login banner file: %v", err) } } @@ -269,6 +269,7 @@ func (c Configuration) AcceptInboundConnection(conn net.Conn, config *ssh.Server logger.Warn(logSender, "", "failed to accept an incoming connection: %v", err) if _, ok := err.(*ssh.ServerAuthError); !ok { logger.ConnectionFailedLog("", utils.GetIPFromRemoteAddress(remoteAddr.String()), "no_auth_tryed", err.Error()) + metrics.AddNoAuthTryed() } return } @@ -348,7 +349,7 @@ func (c Configuration) AcceptInboundConnection(conn net.Conn, config *ssh.Server func (c Configuration) handleSftpConnection(channel ssh.Channel, connection *Connection) { common.Connections.Add(connection) - defer common.Connections.Remove(connection) + defer common.Connections.Remove(connection.GetID()) // Create a new handler for the currently logged in user's server. handler := c.createHandler(connection) diff --git a/sftpd/sftpd_test.go b/sftpd/sftpd_test.go index f5986ce9..5d6c14ca 100644 --- a/sftpd/sftpd_test.go +++ b/sftpd/sftpd_test.go @@ -1405,6 +1405,8 @@ func TestPreLoginUserCreation(t *testing.T) { assert.NoError(t, err) assert.Equal(t, 1, len(users)) user := users[0] + _, err = httpd.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) err = os.RemoveAll(user.GetHomeDir()) assert.NoError(t, err) err = dataprovider.Close() @@ -4567,7 +4569,7 @@ func TestPermChmod(t *testing.T) { assert.NoError(t, err) err = sftpUploadFile(testFilePath, testFileName, testFileSize, client) assert.NoError(t, err) - err = client.Chmod(testFileName, 0666) + err = client.Chmod(testFileName, os.ModePerm) assert.Error(t, err, "chmod without permission should not succeed") err = client.Remove(testFileName) assert.NoError(t, err) @@ -7084,7 +7086,7 @@ func createTestFile(path string, size int64) error { if err != nil { return err } - return ioutil.WriteFile(path, content, 0666) + return ioutil.WriteFile(path, content, os.ModePerm) } func appendToTestFile(path string, size int64) error { @@ -7093,7 +7095,7 @@ func appendToTestFile(path string, size int64) error { if err != nil { return err } - f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0666) + f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, os.ModePerm) if err != nil { return err } diff --git a/sftpd/ssh_cmd.go b/sftpd/ssh_cmd.go index fa0323d2..a183d99f 100644 --- a/sftpd/ssh_cmd.go +++ b/sftpd/ssh_cmd.go @@ -88,7 +88,7 @@ func processSSHCommand(payload []byte, connection *Connection, channel ssh.Chann func (c *sshCommand) handle() error { common.Connections.Add(c.connection) - defer common.Connections.Remove(c.connection) + defer common.Connections.Remove(c.connection.GetID()) c.connection.UpdateLastActivity() if utils.IsStringInSlice(c.command, sshHashCommands) { @@ -354,7 +354,7 @@ func (c *sshCommand) executeSystemCommand(command systemCommand) error { defer stdin.Close() baseTransfer := common.NewBaseTransfer(nil, c.connection.BaseConnection, nil, command.fsPath, sshDestPath, common.TransferUpload, 0, 0, false) - transfer := newTranfer(baseTransfer, nil, nil, remainingQuotaSize) + transfer := newTransfer(baseTransfer, nil, nil, remainingQuotaSize) w, e := transfer.copyFromReaderToWriter(stdin, c.connection.channel) c.connection.Log(logger.LevelDebug, "command: %#v, copy from remote command to sdtin ended, written: %v, "+ @@ -367,7 +367,7 @@ func (c *sshCommand) executeSystemCommand(command systemCommand) error { go func() { baseTransfer := common.NewBaseTransfer(nil, c.connection.BaseConnection, nil, command.fsPath, sshDestPath, common.TransferDownload, 0, 0, false) - transfer := newTranfer(baseTransfer, nil, nil, 0) + transfer := newTransfer(baseTransfer, nil, nil, 0) w, e := transfer.copyFromReaderToWriter(c.connection.channel, stdout) c.connection.Log(logger.LevelDebug, "command: %#v, copy from sdtout to remote command ended, written: %v err: %v", @@ -381,7 +381,7 @@ func (c *sshCommand) executeSystemCommand(command systemCommand) error { go func() { baseTransfer := common.NewBaseTransfer(nil, c.connection.BaseConnection, nil, command.fsPath, sshDestPath, common.TransferDownload, 0, 0, false) - transfer := newTranfer(baseTransfer, nil, nil, 0) + transfer := newTransfer(baseTransfer, nil, nil, 0) w, e := transfer.copyFromReaderToWriter(c.connection.channel.Stderr(), stderr) c.connection.Log(logger.LevelDebug, "command: %#v, copy from sdterr to remote command ended, written: %v err: %v", diff --git a/sftpd/transfer.go b/sftpd/transfer.go index 4c639f10..fadde755 100644 --- a/sftpd/transfer.go +++ b/sftpd/transfer.go @@ -32,7 +32,7 @@ type transfer struct { maxWriteSize int64 } -func newTranfer(baseTransfer *common.BaseTransfer, pipeWriter *vfs.PipeWriter, pipeReader *pipeat.PipeReaderAt, +func newTransfer(baseTransfer *common.BaseTransfer, pipeWriter *vfs.PipeWriter, pipeReader *pipeat.PipeReaderAt, maxWriteSize int64) *transfer { var writer writerAtCloser var reader readerAtCloser diff --git a/sftpgo.json b/sftpgo.json index 05e8974b..cee5b626 100644 --- a/sftpgo.json +++ b/sftpgo.json @@ -30,6 +30,20 @@ ], "keyboard_interactive_auth_hook": "" }, + "ftpd": { + "bind_port": 0, + "bind_address": "", + "banner": "", + "banner_file": "", + "active_transfers_port_non_20": false, + "force_passive_ip": "", + "passive_port_range": { + "start": 50000, + "end": 50100 + }, + "certificate_file": "", + "certificate_key_file": "" + }, "data_provider": { "driver": "sqlite", "name": "sftpgo.db", diff --git a/vfs/osfs.go b/vfs/osfs.go index 20b1adac..ec9b2774 100644 --- a/vfs/osfs.go +++ b/vfs/osfs.go @@ -71,7 +71,7 @@ func (OsFs) Create(name string, flag int) (*os.File, *PipeWriter, func(), error) if flag == 0 { f, err = os.Create(name) } else { - f, err = os.OpenFile(name, flag, 0666) + f, err = os.OpenFile(name, flag, os.ModePerm) } return f, nil, nil, err }