add support for SFTP subsystem mode

Fixes #204

README.md

@@ -44,6 +44,7 @@ It can serve local filesystem, S3 (compatible) Object Storage, Google Cloud Stor
 - [Web based administration interface](./docs/web-admin.md) to easily manage users, folders and connections.
 - Easy [migration](./examples/rest-api-cli#convert-users-from-other-stores) from Linux system user accounts.
 - [Portable mode](./docs/portable-mode.md): a convenient way to share a single directory on demand.
+- [SFTP subsystem mode](./docs/sftp-subsystem.md): you can use SFTPGo as OpenSSH's SFTP subsystem.
 - Performance analysis using built-in [profiler](./docs/profiling.md).
 - Configuration format is at your choice: JSON, TOML, YAML, HCL, envfile are supported.
 - Log files are accurate and they are saved in the easily parsable JSON format ([more information](./docs/logs.md)).

cmd/startsubsys.go

@@ -0,0 +1,167 @@
+package cmd
+
+import (
+	"io"
+	"os"
+	"os/user"
+	"path/filepath"
+
+	"github.com/rs/xid"
+	"github.com/rs/zerolog"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	"github.com/drakkan/sftpgo/common"
+	"github.com/drakkan/sftpgo/config"
+	"github.com/drakkan/sftpgo/dataprovider"
+	"github.com/drakkan/sftpgo/logger"
+	"github.com/drakkan/sftpgo/sftpd"
+	"github.com/drakkan/sftpgo/version"
+)
+
+var (
+	logJournalD     = false
+	preserveHomeDir = false
+	subsystemCmd    = &cobra.Command{
+		Use:   "startsubsys",
+		Short: "Use SFTPGo as SFTP file transfer subsystem",
+		Long: `In this mode SFTPGo speaks the server side of SFTP protocol to stdout and
+expects client requests from stdin.
+This mode is not intended to be called directly, but from sshd using the
+Subsystem option.
+For example adding a line like this one in "/etc/ssh/sshd_config":
+
+Subsystem	sftp	sftpgo startsubsys
+
+Command-line flags should be specified in the Subsystem declaration.
+`,
+		Run: func(cmd *cobra.Command, args []string) {
+			logSender := "startsubsys"
+			connectionID := xid.New().String()
+			logLevel := zerolog.DebugLevel
+			if !logVerbose {
+				logLevel = zerolog.InfoLevel
+			}
+			if logJournalD {
+				logger.InitJournalDLogger(logLevel)
+			} else {
+				logger.InitStdErrLogger(logLevel)
+			}
+			osUser, err := user.Current()
+			if err != nil {
+				logger.Error(logSender, connectionID, "unable to get the current user: %v", err)
+				os.Exit(1)
+			}
+			username := osUser.Username
+			homedir := osUser.HomeDir
+			logger.Info(logSender, connectionID, "starting SFTPGo %v as subsystem, user %#v home dir %#v config dir %#v",
+				version.Get(), username, homedir, configDir)
+			err = config.LoadConfig(configDir, configFile)
+			if err != nil {
+				logger.Error(logSender, connectionID, "unable to load configuration: %v", err)
+				os.Exit(1)
+			}
+			commonConfig := config.GetCommonConfig()
+			// idle connection are managed externally
+			commonConfig.IdleTimeout = 0
+			config.SetCommonConfig(commonConfig)
+			common.Initialize(config.GetCommonConfig())
+			dataProviderConf := config.GetProviderConf()
+			if dataProviderConf.Driver == dataprovider.SQLiteDataProviderName || dataProviderConf.Driver == dataprovider.BoltDataProviderName {
+				logger.Debug(logSender, connectionID, "data provider %#v not supported in subsystem mode, using %#v provider",
+					dataProviderConf.Driver, dataprovider.MemoryDataProviderName)
+				dataProviderConf.Driver = dataprovider.MemoryDataProviderName
+				dataProviderConf.Name = ""
+				dataProviderConf.PreferDatabaseCredentials = true
+			}
+			config.SetProviderConf(dataProviderConf)
+			err = dataprovider.Initialize(dataProviderConf, configDir)
+			if err != nil {
+				logger.Error(logSender, connectionID, "unable to initialize the data provider: %v", err)
+				os.Exit(1)
+			}
+			httpConfig := config.GetHTTPConfig()
+			httpConfig.Initialize(configDir)
+			user, err := dataprovider.UserExists(username)
+			if err == nil {
+				if user.HomeDir != filepath.Clean(homedir) && !preserveHomeDir {
+					// update the user
+					user.HomeDir = filepath.Clean(homedir)
+					err = dataprovider.UpdateUser(user)
+					if err != nil {
+						logger.Error(logSender, connectionID, "unable to update user %#v: %v", username, err)
+						os.Exit(1)
+					}
+				}
+			} else {
+				user.Username = username
+				user.HomeDir = homedir
+				user.Password = connectionID
+				user.Permissions = make(map[string][]string)
+				user.Permissions["/"] = []string{dataprovider.PermAny}
+				err = dataprovider.AddUser(user)
+				if err != nil {
+					logger.Error(logSender, connectionID, "unable to add user %#v: %v", username, err)
+					os.Exit(1)
+				}
+			}
+			err = sftpd.ServeSubSystemConnection(user, connectionID, os.Stdin, os.Stdout)
+			if err != nil && err != io.EOF {
+				logger.Warn(logSender, connectionID, "serving subsystem finished with error: %v", err)
+				os.Exit(1)
+			}
+			logger.Info(logSender, connectionID, "serving subsystem finished")
+			os.Exit(0)
+		},
+	}
+)
+
+func init() {
+	subsystemCmd.Flags().BoolVarP(&preserveHomeDir, "preserve-home", "p", false, `If the user already exists, the existing home
+directory will not be changed`)
+	subsystemCmd.Flags().BoolVarP(&logJournalD, "log-to-journald", "j", false, `Send logs to journald. Only available on Linux.
+Use:
+
+$ journalctl -o verbose -f
+
+To see full logs.
+If not set, the logs will be sent to the standard
+error`)
+	viper.SetDefault(configDirKey, defaultConfigDir)
+	viper.BindEnv(configDirKey, "SFTPGO_CONFIG_DIR") //nolint:errcheck // err is not nil only if the key to bind is missing
+	subsystemCmd.Flags().StringVarP(&configDir, configDirFlag, "c", viper.GetString(configDirKey),
+		`Location for SFTPGo config dir. This directory
+should contain the "sftpgo" configuration file
+or the configured config-file and it is used as
+the base for files with a relative path (eg. the
+private keys for the SFTP server, the SQLite
+database if you use SQLite as data provider).
+This flag can be set using SFTPGO_CONFIG_DIR
+env var too.`)
+	viper.BindPFlag(configDirKey, subsystemCmd.Flags().Lookup(configDirFlag)) //nolint:errcheck
+
+	viper.SetDefault(configFileKey, defaultConfigName)
+	viper.BindEnv(configFileKey, "SFTPGO_CONFIG_FILE") //nolint:errcheck
+	subsystemCmd.Flags().StringVarP(&configFile, configFileFlag, "f", viper.GetString(configFileKey),
+		`Name for SFTPGo configuration file. It must be
+the name of a file stored in config-dir not the
+absolute path to the configuration file. The
+specified file name must have no extension we
+automatically load JSON, YAML, TOML, HCL and
+Java properties. Therefore if you set "sftpgo"
+then "sftpgo.json", "sftpgo.yaml" and so on
+are searched.
+This flag can be set using SFTPGO_CONFIG_FILE
+env var too.`)
+	viper.BindPFlag(configFileKey, subsystemCmd.Flags().Lookup(configFileFlag)) //nolint:errcheck
+
+	viper.SetDefault(logVerboseKey, defaultLogVerbose)
+	viper.BindEnv(logVerboseKey, "SFTPGO_LOG_VERBOSE") //nolint:errcheck
+	subsystemCmd.Flags().BoolVarP(&logVerbose, logVerboseFlag, "v", viper.GetBool(logVerboseKey),
+		`Enable verbose logs. This flag can be set
+using SFTPGO_LOG_VERBOSE env var too.
+`)
+	viper.BindPFlag(logVerboseKey, subsystemCmd.Flags().Lookup(logVerboseFlag)) //nolint:errcheck
+
+	rootCmd.AddCommand(subsystemCmd)
+}

dataprovider/dataprovider.go

@@ -384,8 +384,10 @@ func Initialize(cnf Config, basePath string) error {
 	if err = validateHooks(); err != nil {
 		return err
 	}
-	if err = validateCredentialsDir(basePath); err != nil {
-		return err
+	if !cnf.PreferDatabaseCredentials {
+		if err = validateCredentialsDir(basePath); err != nil {
+			return err
+		}
 	}
 	err = createProvider(basePath)
 	if err != nil {

docs/sftp-subsystem.md

@@ -0,0 +1,57 @@
+# SFTP subsystem mode
+
+In this mode SFTPGo speaks the server side of SFTP protocol to stdout and expects client requests from stdin.
+You can use SFTPGo as subsystem via the `startsubsys` command.
+This mode is not intended to be called directly, but from sshd using the `Subsystem` option.
+For example adding a line like this one in `/etc/ssh/sshd_config`:
+
+```shell
+Subsystem    sftp    sftpgo startsubsys
+```
+
+Command-line flags should be specified in the Subsystem declaration.
+
+```shell
+Usage:
+  sftpgo startsubsys [flags]
+
+Flags:
+  -c, --config-dir string    Location for SFTPGo config dir. This directory
+                             should contain the "sftpgo" configuration file
+                             or the configured config-file and it is used as
+                             the base for files with a relative path (eg. the
+                             private keys for the SFTP server, the SQLite
+                             database if you use SQLite as data provider).
+                             This flag can be set using SFTPGO_CONFIG_DIR
+                             env var too. (default ".")
+  -f, --config-file string   Name for SFTPGo configuration file. It must be
+                             the name of a file stored in config-dir not the
+                             absolute path to the configuration file. The
+                             specified file name must have no extension we
+                             automatically load JSON, YAML, TOML, HCL and
+                             Java properties. Therefore if you set "sftpgo"
+                             then "sftpgo.json", "sftpgo.yaml" and so on
+                             are searched.
+                             This flag can be set using SFTPGO_CONFIG_FILE
+                             env var too. (default "sftpgo")
+  -h, --help                 help for startsubsys
+  -j, --log-to-journald      Send logs to journald. Only available on Linux.
+                             Use:
+
+                             $ journalctl -o verbose -f
+
+                             To see full logs.
+                             If not set, the logs will be sent to the standard
+                             error
+  -v, --log-verbose          Enable verbose logs. This flag can be set
+                             using SFTPGO_LOG_VERBOSE env var too.
+                              (default true)
+  -p, --preserve-home        If the user already exists, the existing home
+                             directory will not be changed
+```
+
+In this mode `bolt` and `sqlite` providers are not usable as the same database file cannot be shared among multiple processes, if one of these provider is configured it will be automatically changed to `memory` provider.
+
+The username and home directory for the logged in user are determined using [user.Current()](https://golang.org/pkg/os/user/#Current).
+If the user who is logging is not found within the SFTPGo data provider, it is added automatically.
+You can pre-configure the users inside the SFTPGo data provider, this way you can use a different home directory, restrict permissions and such.

examples/ldapauthserver/logger/request_logger.go

@@ -58,7 +58,7 @@ func (l *StructuredLoggerEntry) Write(status, bytes int, header http.Header, ela
 		Int("resp_status", status).
 		Int("resp_size", bytes).
 		Int64("elapsed_ms", elapsed.Nanoseconds()/1000000).
-		Msg("")
+		Send()
 }
 
 // Panic logs panics
@@ -69,5 +69,5 @@ func (l *StructuredLoggerEntry) Panic(v interface{}, stack []byte) {
 		Fields(l.fields).
 		Str("stack", string(stack)).
 		Str("panic", fmt.Sprintf("%+v", v)).
-		Msg("")
+		Send()
 }

go.sum

@@ -106,6 +106,7 @@ github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=

logger/journald.go

@@ -0,0 +1,14 @@
+// +build linux
+
+package logger
+
+import (
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/journald"
+)
+
+// InitJournalDLogger configures the logger to write to journald
+func InitJournalDLogger(level zerolog.Level) {
+	logger = zerolog.New(journald.NewJournalDWriter()).Level(level)
+	consoleLogger = zerolog.Nop()
+}

logger/journald_nolinux.go

@@ -0,0 +1,10 @@
+// +build !linux
+
+package logger
+
+import "github.com/rs/zerolog"
+
+// InitJournalDLogger configures the logger to write to journald
+func InitJournalDLogger(level zerolog.Level) {
+	InitStdErrLogger(level)
+}

logger/logger.go

@@ -67,6 +67,14 @@ func InitLogger(logFilePath string, logMaxSize int, logMaxBackups int, logMaxAge
 	logger = logger.Level(level)
 }
 
+// InitStdErrLogger configures the logger to write to stderr
+func InitStdErrLogger(level zerolog.Level) {
+	logger = zerolog.New(&logSyncWrapper{
+		output: os.Stderr,
+	}).Level(level)
+	consoleLogger = zerolog.Nop()
+}
+
 // DisableLogger disable the main logger.
 // ConsoleLogger will not be affected
 func DisableLogger() {
@@ -163,7 +171,7 @@ func TransferLog(operation string, path string, elapsed int64, size int64, user
 		Str("file_path", path).
 		Str("connection_id", connectionID).
 		Str("protocol", protocol).
-		Msg("")
+		Send()
 }
 
 // CommandLog logs an SFTP/SCP/SSH command
@@ -184,7 +192,7 @@ func CommandLog(command, path, target, user, fileMode, connectionID, protocol st
 		Str("ssh_command", sshCommand).
 		Str("connection_id", connectionID).
 		Str("protocol", protocol).
-		Msg("")
+		Send()
 }
 
 // ConnectionFailedLog logs failed attempts to initialize a connection.
@@ -200,7 +208,7 @@ func ConnectionFailedLog(user, ip, loginType, protocol, errorString string) {
 		Str("login_type", loginType).
 		Str("protocol", protocol).
 		Str("error", errorString).
-		Msg("")
+		Send()
 }
 
 func isLogFilePathValid(logFilePath string) bool {

logger/request_logger.go

@@ -64,7 +64,7 @@ func (l *StructuredLoggerEntry) Write(status, bytes int, header http.Header, ela
 		Int("resp_status", status).
 		Int("resp_size", bytes).
 		Int64("elapsed_ms", elapsed.Nanoseconds()/1000000).
-		Msg("")
+		Send()
 }
 
 // Panic logs panics
@@ -75,5 +75,5 @@ func (l *StructuredLoggerEntry) Panic(v interface{}, stack []byte) {
 		Fields(l.fields).
 		Str("stack", string(stack)).
 		Str("panic", fmt.Sprintf("%+v", v)).
-		Msg("")
+		Send()
 }

sftpd/handler.go

@@ -8,7 +8,6 @@ import (
 	"time"
 
 	"github.com/pkg/sftp"
-	"golang.org/x/crypto/ssh"
 
 	"github.com/drakkan/sftpgo/common"
 	"github.com/drakkan/sftpgo/dataprovider"
@@ -23,7 +22,7 @@ type Connection struct {
 	ClientVersion string
 	// Remote address for this connection
 	RemoteAddr net.Addr
-	channel    ssh.Channel
+	channel    io.ReadWriteCloser
 	command    string
 }
 

sftpd/internal_test.go

@@ -1814,3 +1814,35 @@ func TestRecursiveCopyErrors(t *testing.T) {
 	err = sshCmd.checkRecursiveCopyPermissions("adir", "another", "/another")
 	assert.Error(t, err)
 }
+
+func TestSFTPSubSystem(t *testing.T) {
+	permissions := make(map[string][]string)
+	permissions["/"] = []string{dataprovider.PermAny}
+	user := dataprovider.User{
+		Permissions: permissions,
+		HomeDir:     os.TempDir(),
+	}
+	user.FsConfig.Provider = dataprovider.AzureBlobFilesystemProvider
+	err := ServeSubSystemConnection(user, "connID", nil, nil)
+	assert.Error(t, err)
+	user.FsConfig.Provider = dataprovider.LocalFilesystemProvider
+
+	buf := make([]byte, 0, 4096)
+	stdErrBuf := make([]byte, 0, 4096)
+	mockSSHChannel := &MockChannel{
+		Buffer:       bytes.NewBuffer(buf),
+		StdErrBuffer: bytes.NewBuffer(stdErrBuf),
+	}
+	// this is 327680 and it will result in packet too long error
+	_, err = mockSSHChannel.Write([]byte{0x00, 0x05, 0x00, 0x00, 0x00, 0x00})
+	assert.NoError(t, err)
+	err = ServeSubSystemConnection(user, "id", mockSSHChannel, mockSSHChannel)
+	assert.EqualError(t, err, "packet too long")
+
+	subsystemChannel := newSubsystemChannel(mockSSHChannel, mockSSHChannel)
+	n, err := subsystemChannel.Write([]byte{0x00})
+	assert.NoError(t, err)
+	assert.Equal(t, n, 1)
+	err = subsystemChannel.Close()
+	assert.NoError(t, err)
+}

sftpd/ssh_cmd.go

@@ -381,7 +381,7 @@ func (c *sshCommand) executeSystemCommand(command systemCommand) error {
 			common.TransferDownload, 0, 0, 0, false, c.connection.Fs)
 		transfer := newTransfer(baseTransfer, nil, nil, nil)
 
-		w, e := transfer.copyFromReaderToWriter(c.connection.channel.Stderr(), stderr)
+		w, e := transfer.copyFromReaderToWriter(c.connection.channel.(ssh.Channel).Stderr(), stderr)
 		c.connection.Log(logger.LevelDebug, "command: %#v, copy from sdterr to remote command ended, written: %v err: %v",
 			c.connection.command, w, e)
 		// os.ErrClosed means that the command is finished so we don't need to do anything
@@ -707,7 +707,7 @@ func (c *sshCommand) sendExitStatus(err error) {
 	exitStatus := sshSubsystemExitStatus{
 		Status: status,
 	}
-	c.connection.channel.SendRequest("exit-status", false, ssh.Marshal(&exitStatus)) //nolint:errcheck
+	c.connection.channel.(ssh.Channel).SendRequest("exit-status", false, ssh.Marshal(&exitStatus)) //nolint:errcheck
 	c.connection.channel.Close()
 	// for scp we notify single uploads/downloads
 	if c.command != scpCmdName {

sftpd/subsystem.go

@@ -0,0 +1,63 @@
+package sftpd
+
+import (
+	"io"
+	"net"
+
+	"github.com/pkg/sftp"
+
+	"github.com/drakkan/sftpgo/common"
+	"github.com/drakkan/sftpgo/dataprovider"
+)
+
+type subsystemChannel struct {
+	reader io.Reader
+	writer io.Writer
+}
+
+func (s *subsystemChannel) Read(p []byte) (int, error) {
+	return s.reader.Read(p)
+}
+
+func (s *subsystemChannel) Write(p []byte) (int, error) {
+	return s.writer.Write(p)
+}
+
+func (s *subsystemChannel) Close() error {
+	return nil
+}
+
+func newSubsystemChannel(reader io.Reader, writer io.Writer) *subsystemChannel {
+	return &subsystemChannel{
+		reader: reader,
+		writer: writer,
+	}
+}
+
+// ServeSubSystemConnection handles a connection as SSH subsystem
+func ServeSubSystemConnection(user dataprovider.User, connectionID string, reader io.Reader, writer io.Writer) error {
+	fs, err := user.GetFilesystem(connectionID)
+	if err != nil {
+		return err
+	}
+	fs.CheckRootPath(user.Username, user.GetUID(), user.GetGID())
+	dataprovider.UpdateLastLogin(user) //nolint:errcheck
+
+	connection := &Connection{
+		BaseConnection: common.NewBaseConnection(fs.ConnectionID(), common.ProtocolSFTP, user, fs),
+		ClientVersion:  "",
+		RemoteAddr:     &net.IPAddr{},
+		channel:        newSubsystemChannel(reader, writer),
+	}
+	common.Connections.Add(connection)
+	defer common.Connections.Remove(connection.GetID())
+
+	server := sftp.NewRequestServer(connection.channel, sftp.Handlers{
+		FileGet:  connection,
+		FilePut:  connection,
+		FileCmd:  connection,
+		FileList: connection,
+	}, sftp.WithRSAllocator())
+
+	return server.Serve()
+}

webdavd/server.go

@@ -255,7 +255,7 @@ func writeLog(r *http.Request, err error) {
 		Str("sender", logSender).
 		Fields(fields).
 		Err(err).
-		Msg("")
+		Send()
 }
 
 func checkRemoteAddress(r *http.Request) {