// Package vfs provides local and remote filesystems support package vfs import ( "errors" "fmt" "io" "net/url" "os" "path" "path/filepath" "runtime" "strings" "time" "github.com/eikenb/pipeat" "github.com/pkg/sftp" "github.com/drakkan/sftpgo/v2/kms" "github.com/drakkan/sftpgo/v2/logger" "github.com/drakkan/sftpgo/v2/sdk" "github.com/drakkan/sftpgo/v2/util" ) const dirMimeType = "inode/directory" var ( validAzAccessTier = []string{"", "Archive", "Hot", "Cool"} // ErrStorageSizeUnavailable is returned if the storage backend does not support getting the size ErrStorageSizeUnavailable = errors.New("unable to get available size for this storage backend") // ErrVfsUnsupported defines the error for an unsupported VFS operation ErrVfsUnsupported = errors.New("not supported") credentialsDirPath string tempPath string sftpFingerprints []string ) // SetCredentialsDirPath sets the credentials dir path func SetCredentialsDirPath(credentialsPath string) { credentialsDirPath = credentialsPath } // GetCredentialsDirPath returns the credentials dir path func GetCredentialsDirPath() string { return credentialsDirPath } // SetTempPath sets the path for temporary files func SetTempPath(fsPath string) { tempPath = fsPath } // GetTempPath returns the path for temporary files func GetTempPath() string { return tempPath } // SetSFTPFingerprints sets the SFTP host key fingerprints func SetSFTPFingerprints(fp []string) { sftpFingerprints = fp } // Fs defines the interface for filesystem backends type Fs interface { Name() string ConnectionID() string Stat(name string) (os.FileInfo, error) Lstat(name string) (os.FileInfo, error) Open(name string, offset int64) (File, *pipeat.PipeReaderAt, func(), error) Create(name string, flag int) (File, *PipeWriter, func(), error) Rename(source, target string) error Remove(name string, isDir bool) error Mkdir(name string) error MkdirAll(name string, uid int, gid int) error Symlink(source, target string) error Chown(name string, uid int, gid int) error Chmod(name string, mode os.FileMode) error Chtimes(name string, atime, mtime time.Time) error Truncate(name string, size int64) error ReadDir(dirname string) ([]os.FileInfo, error) Readlink(name string) (string, error) IsUploadResumeSupported() bool IsAtomicUploadSupported() bool CheckRootPath(username string, uid int, gid int) bool ResolvePath(sftpPath string) (string, error) IsNotExist(err error) bool IsPermission(err error) bool IsNotSupported(err error) bool ScanRootDirContents() (int, int64, error) GetDirSize(dirname string) (int, int64, error) GetAtomicUploadPath(name string) string GetRelativePath(name string) string Walk(root string, walkFn filepath.WalkFunc) error Join(elem ...string) string HasVirtualFolders() bool GetMimeType(name string) (string, error) GetAvailableDiskSize(dirName string) (*sftp.StatVFS, error) Close() error } // File defines an interface representing a SFTPGo file type File interface { io.Reader io.Writer io.Closer io.ReaderAt io.WriterAt io.Seeker Stat() (os.FileInfo, error) Name() string Truncate(size int64) error } // QuotaCheckResult defines the result for a quota check type QuotaCheckResult struct { HasSpace bool AllowedSize int64 AllowedFiles int UsedSize int64 UsedFiles int QuotaSize int64 QuotaFiles int } // GetRemainingSize returns the remaining allowed size func (q *QuotaCheckResult) GetRemainingSize() int64 { if q.QuotaSize > 0 { return q.QuotaSize - q.UsedSize } return 0 } // GetRemainingFiles returns the remaining allowed files func (q *QuotaCheckResult) GetRemainingFiles() int { if q.QuotaFiles > 0 { return q.QuotaFiles - q.UsedFiles } return 0 } // S3FsConfig defines the configuration for S3 based filesystem type S3FsConfig struct { sdk.S3FsConfig } func (c *S3FsConfig) isEqual(other *S3FsConfig) bool { if c.Bucket != other.Bucket { return false } if c.KeyPrefix != other.KeyPrefix { return false } if c.Region != other.Region { return false } if c.AccessKey != other.AccessKey { return false } if c.Endpoint != other.Endpoint { return false } if c.StorageClass != other.StorageClass { return false } if c.UploadPartSize != other.UploadPartSize { return false } if c.UploadConcurrency != other.UploadConcurrency { return false } if c.AccessSecret == nil { c.AccessSecret = kms.NewEmptySecret() } if other.AccessSecret == nil { other.AccessSecret = kms.NewEmptySecret() } return c.AccessSecret.IsEqual(other.AccessSecret) } func (c *S3FsConfig) checkCredentials() error { if c.AccessKey == "" && !c.AccessSecret.IsEmpty() { return errors.New("access_key cannot be empty with access_secret not empty") } if c.AccessSecret.IsEmpty() && c.AccessKey != "" { return errors.New("access_secret cannot be empty with access_key not empty") } if c.AccessSecret.IsEncrypted() && !c.AccessSecret.IsValid() { return errors.New("invalid encrypted access_secret") } if !c.AccessSecret.IsEmpty() && !c.AccessSecret.IsValidInput() { return errors.New("invalid access_secret") } return nil } // EncryptCredentials encrypts access secret if it is in plain text func (c *S3FsConfig) EncryptCredentials(additionalData string) error { if c.AccessSecret.IsPlain() { c.AccessSecret.SetAdditionalData(additionalData) err := c.AccessSecret.Encrypt() if err != nil { return err } } return nil } // Validate returns an error if the configuration is not valid func (c *S3FsConfig) Validate() error { if c.AccessSecret == nil { c.AccessSecret = kms.NewEmptySecret() } if c.Bucket == "" { return errors.New("bucket cannot be empty") } if c.Region == "" { return errors.New("region cannot be empty") } if err := c.checkCredentials(); err != nil { return err } if c.KeyPrefix != "" { if strings.HasPrefix(c.KeyPrefix, "/") { return errors.New("key_prefix cannot start with /") } c.KeyPrefix = path.Clean(c.KeyPrefix) if !strings.HasSuffix(c.KeyPrefix, "/") { c.KeyPrefix += "/" } } if c.UploadPartSize != 0 && (c.UploadPartSize < 5 || c.UploadPartSize > 5000) { return errors.New("upload_part_size cannot be != 0, lower than 5 (MB) or greater than 5000 (MB)") } if c.UploadConcurrency < 0 || c.UploadConcurrency > 64 { return fmt.Errorf("invalid upload concurrency: %v", c.UploadConcurrency) } return nil } // GCSFsConfig defines the configuration for Google Cloud Storage based filesystem type GCSFsConfig struct { sdk.GCSFsConfig } func (c *GCSFsConfig) isEqual(other *GCSFsConfig) bool { if c.Bucket != other.Bucket { return false } if c.KeyPrefix != other.KeyPrefix { return false } if c.AutomaticCredentials != other.AutomaticCredentials { return false } if c.StorageClass != other.StorageClass { return false } if c.Credentials == nil { c.Credentials = kms.NewEmptySecret() } if other.Credentials == nil { other.Credentials = kms.NewEmptySecret() } return c.Credentials.IsEqual(other.Credentials) } // Validate returns an error if the configuration is not valid func (c *GCSFsConfig) Validate(credentialsFilePath string) error { if c.Credentials == nil { c.Credentials = kms.NewEmptySecret() } if c.Bucket == "" { return errors.New("bucket cannot be empty") } if c.KeyPrefix != "" { if strings.HasPrefix(c.KeyPrefix, "/") { return errors.New("key_prefix cannot start with /") } c.KeyPrefix = path.Clean(c.KeyPrefix) if !strings.HasSuffix(c.KeyPrefix, "/") { c.KeyPrefix += "/" } } if c.Credentials.IsEncrypted() && !c.Credentials.IsValid() { return errors.New("invalid encrypted credentials") } if !c.Credentials.IsValidInput() && c.AutomaticCredentials == 0 { fi, err := os.Stat(credentialsFilePath) if err != nil { return fmt.Errorf("invalid credentials %v", err) } if fi.Size() == 0 { return errors.New("credentials cannot be empty") } } return nil } // AzBlobFsConfig defines the configuration for Azure Blob Storage based filesystem type AzBlobFsConfig struct { sdk.AzBlobFsConfig } func (c *AzBlobFsConfig) isEqual(other *AzBlobFsConfig) bool { if c.Container != other.Container { return false } if c.AccountName != other.AccountName { return false } if c.Endpoint != other.Endpoint { return false } if c.SASURL.IsEmpty() { c.SASURL = kms.NewEmptySecret() } if other.SASURL.IsEmpty() { other.SASURL = kms.NewEmptySecret() } if !c.SASURL.IsEqual(other.SASURL) { return false } if c.KeyPrefix != other.KeyPrefix { return false } if c.UploadPartSize != other.UploadPartSize { return false } if c.UploadConcurrency != other.UploadConcurrency { return false } if c.UseEmulator != other.UseEmulator { return false } if c.AccessTier != other.AccessTier { return false } if c.AccountKey == nil { c.AccountKey = kms.NewEmptySecret() } if other.AccountKey == nil { other.AccountKey = kms.NewEmptySecret() } return c.AccountKey.IsEqual(other.AccountKey) } // EncryptCredentials encrypts access secret if it is in plain text func (c *AzBlobFsConfig) EncryptCredentials(additionalData string) error { if c.AccountKey.IsPlain() { c.AccountKey.SetAdditionalData(additionalData) if err := c.AccountKey.Encrypt(); err != nil { return err } } if c.SASURL.IsPlain() { c.SASURL.SetAdditionalData(additionalData) if err := c.SASURL.Encrypt(); err != nil { return err } } return nil } func (c *AzBlobFsConfig) checkCredentials() error { if c.SASURL.IsPlain() { _, err := url.Parse(c.SASURL.GetPayload()) return err } if c.SASURL.IsEncrypted() && !c.SASURL.IsValid() { return errors.New("invalid encrypted sas_url") } if !c.SASURL.IsEmpty() { return nil } if c.AccountName == "" || !c.AccountKey.IsValidInput() { return errors.New("credentials cannot be empty or invalid") } if c.AccountKey.IsEncrypted() && !c.AccountKey.IsValid() { return errors.New("invalid encrypted account_key") } return nil } // Validate returns an error if the configuration is not valid func (c *AzBlobFsConfig) Validate() error { if c.AccountKey == nil { c.AccountKey = kms.NewEmptySecret() } if c.SASURL == nil { c.SASURL = kms.NewEmptySecret() } // container could be embedded within SAS URL we check this at runtime if c.SASURL.IsEmpty() && c.Container == "" { return errors.New("container cannot be empty") } if err := c.checkCredentials(); err != nil { return err } if c.KeyPrefix != "" { if strings.HasPrefix(c.KeyPrefix, "/") { return errors.New("key_prefix cannot start with /") } c.KeyPrefix = path.Clean(c.KeyPrefix) if !strings.HasSuffix(c.KeyPrefix, "/") { c.KeyPrefix += "/" } } if c.UploadPartSize < 0 || c.UploadPartSize > 100 { return fmt.Errorf("invalid upload part size: %v", c.UploadPartSize) } if c.UploadConcurrency < 0 || c.UploadConcurrency > 64 { return fmt.Errorf("invalid upload concurrency: %v", c.UploadConcurrency) } if !util.IsStringInSlice(c.AccessTier, validAzAccessTier) { return fmt.Errorf("invalid access tier %#v, valid values: \"''%v\"", c.AccessTier, strings.Join(validAzAccessTier, ", ")) } return nil } // CryptFsConfig defines the configuration to store local files as encrypted type CryptFsConfig struct { sdk.CryptFsConfig } func (c *CryptFsConfig) isEqual(other *CryptFsConfig) bool { if c.Passphrase == nil { c.Passphrase = kms.NewEmptySecret() } if other.Passphrase == nil { other.Passphrase = kms.NewEmptySecret() } return c.Passphrase.IsEqual(other.Passphrase) } // EncryptCredentials encrypts access secret if it is in plain text func (c *CryptFsConfig) EncryptCredentials(additionalData string) error { if c.Passphrase.IsPlain() { c.Passphrase.SetAdditionalData(additionalData) if err := c.Passphrase.Encrypt(); err != nil { return err } } return nil } // Validate returns an error if the configuration is not valid func (c *CryptFsConfig) Validate() error { if c.Passphrase == nil || c.Passphrase.IsEmpty() { return errors.New("invalid passphrase") } if !c.Passphrase.IsValidInput() { return errors.New("passphrase cannot be empty or invalid") } if c.Passphrase.IsEncrypted() && !c.Passphrase.IsValid() { return errors.New("invalid encrypted passphrase") } return nil } // PipeWriter defines a wrapper for pipeat.PipeWriterAt. type PipeWriter struct { writer *pipeat.PipeWriterAt err error done chan bool } // NewPipeWriter initializes a new PipeWriter func NewPipeWriter(w *pipeat.PipeWriterAt) *PipeWriter { return &PipeWriter{ writer: w, err: nil, done: make(chan bool), } } // Close waits for the upload to end, closes the pipeat.PipeWriterAt and returns an error if any. func (p *PipeWriter) Close() error { p.writer.Close() //nolint:errcheck // the returned error is always null <-p.done return p.err } // Done unlocks other goroutines waiting on Close(). // It must be called when the upload ends func (p *PipeWriter) Done(err error) { p.err = err p.done <- true } // WriteAt is a wrapper for pipeat WriteAt func (p *PipeWriter) WriteAt(data []byte, off int64) (int, error) { return p.writer.WriteAt(data, off) } // Write is a wrapper for pipeat Write func (p *PipeWriter) Write(data []byte) (int, error) { return p.writer.Write(data) } // IsDirectory checks if a path exists and is a directory func IsDirectory(fs Fs, path string) (bool, error) { fileInfo, err := fs.Stat(path) if err != nil { return false, err } return fileInfo.IsDir(), err } // IsLocalOsFs returns true if fs is a local filesystem implementation func IsLocalOsFs(fs Fs) bool { return fs.Name() == osFsName } // IsCryptOsFs returns true if fs is an encrypted local filesystem implementation func IsCryptOsFs(fs Fs) bool { return fs.Name() == cryptFsName } // IsSFTPFs returns true if fs is an SFTP filesystem func IsSFTPFs(fs Fs) bool { return strings.HasPrefix(fs.Name(), sftpFsName) } // IsBufferedSFTPFs returns true if this is a buffered SFTP filesystem func IsBufferedSFTPFs(fs Fs) bool { if !IsSFTPFs(fs) { return false } return !fs.IsUploadResumeSupported() } // IsLocalOrUnbufferedSFTPFs returns true if fs is local or SFTP with no buffer func IsLocalOrUnbufferedSFTPFs(fs Fs) bool { if IsLocalOsFs(fs) { return true } if IsSFTPFs(fs) { return fs.IsUploadResumeSupported() } return false } // IsLocalOrSFTPFs returns true if fs is local or SFTP func IsLocalOrSFTPFs(fs Fs) bool { return IsLocalOsFs(fs) || IsSFTPFs(fs) } // HasOpenRWSupport returns true if the fs can open a file // for reading and writing at the same time func HasOpenRWSupport(fs Fs) bool { if IsLocalOsFs(fs) { return true } if IsSFTPFs(fs) && fs.IsUploadResumeSupported() { return true } return false } // IsLocalOrCryptoFs returns true if fs is local or local encrypted func IsLocalOrCryptoFs(fs Fs) bool { return IsLocalOsFs(fs) || IsCryptOsFs(fs) } // SetPathPermissions calls fs.Chown. // It does nothing for local filesystem on windows func SetPathPermissions(fs Fs, path string, uid int, gid int) { if uid == -1 && gid == -1 { return } if IsLocalOsFs(fs) { if runtime.GOOS == "windows" { return } } if err := fs.Chown(path, uid, gid); err != nil { fsLog(fs, logger.LevelWarn, "error chowning path %v: %v", path, err) } } func fsLog(fs Fs, level logger.LogLevel, format string, v ...interface{}) { logger.Log(level, fs.Name(), fs.ConnectionID(), format, v...) }