+
+
+
+
+
+
+
@@ -295,6 +333,22 @@
}
return true;
});
+
+ onFilesystemChanged('{{.User.FsConfig.Provider}}');
+
});
+
+ function onFilesystemChanged(val){
+ if (val == '1'){
+ $('.form-group.row.gcs').hide();
+ $('.form-group.row.s3').show();
+ } else if (val == '2'){
+ $('.form-group.row.gcs').show();
+ $('.form-group.row.s3').hide();
+ } else {
+ $('.form-group.row.gcs').hide();
+ $('.form-group.row.s3').hide();
+ }
+ }
{{end}}
\ No newline at end of file
diff --git a/vfs/s3fileinfo.go b/vfs/fileinfo.go
similarity index 62%
rename from vfs/s3fileinfo.go
rename to vfs/fileinfo.go
index 2cdab62c..aaaa4fcf 100644
--- a/vfs/s3fileinfo.go
+++ b/vfs/fileinfo.go
@@ -5,8 +5,8 @@ import (
"time"
)
-// S3FileInfo implements os.FileInfo for a file in S3.
-type S3FileInfo struct {
+// FileInfo implements os.FileInfo for a file in S3.
+type FileInfo struct {
name string
sizeInBytes int64
modTime time.Time
@@ -14,14 +14,14 @@ type S3FileInfo struct {
sys interface{}
}
-// NewS3FileInfo creates file info.
-func NewS3FileInfo(name string, isDirectory bool, sizeInBytes int64, modTime time.Time) S3FileInfo {
+// NewFileInfo creates file info.
+func NewFileInfo(name string, isDirectory bool, sizeInBytes int64, modTime time.Time) FileInfo {
mode := os.FileMode(0644)
if isDirectory {
mode = os.FileMode(0755) | os.ModeDir
}
- return S3FileInfo{
+ return FileInfo{
name: name,
sizeInBytes: sizeInBytes,
modTime: modTime,
@@ -30,31 +30,31 @@ func NewS3FileInfo(name string, isDirectory bool, sizeInBytes int64, modTime tim
}
// Name provides the base name of the file.
-func (fi S3FileInfo) Name() string {
+func (fi FileInfo) Name() string {
return fi.name
}
// Size provides the length in bytes for a file.
-func (fi S3FileInfo) Size() int64 {
+func (fi FileInfo) Size() int64 {
return fi.sizeInBytes
}
// Mode provides the file mode bits
-func (fi S3FileInfo) Mode() os.FileMode {
+func (fi FileInfo) Mode() os.FileMode {
return fi.mode
}
// ModTime provides the last modification time.
-func (fi S3FileInfo) ModTime() time.Time {
+func (fi FileInfo) ModTime() time.Time {
return fi.modTime
}
// IsDir provides the abbreviation for Mode().IsDir()
-func (fi S3FileInfo) IsDir() bool {
+func (fi FileInfo) IsDir() bool {
return fi.mode&os.ModeDir != 0
}
// Sys provides the underlying data source (can return nil)
-func (fi S3FileInfo) Sys() interface{} {
+func (fi FileInfo) Sys() interface{} {
return fi.getFileInfoSys()
}
diff --git a/vfs/s3fileinfo_unix.go b/vfs/fileinfo_unix.go
similarity index 86%
rename from vfs/s3fileinfo_unix.go
rename to vfs/fileinfo_unix.go
index 375efe0b..943fa219 100644
--- a/vfs/s3fileinfo_unix.go
+++ b/vfs/fileinfo_unix.go
@@ -21,7 +21,7 @@ func init() {
}
}
-func (fi S3FileInfo) getFileInfoSys() interface{} {
+func (fi FileInfo) getFileInfoSys() interface{} {
return &syscall.Stat_t{
Uid: uint32(defaultUID),
Gid: uint32(defaultGID)}
diff --git a/vfs/s3fileinfo_windows.go b/vfs/fileinfo_windows.go
similarity index 58%
rename from vfs/s3fileinfo_windows.go
rename to vfs/fileinfo_windows.go
index 3707a4ba..9947677c 100644
--- a/vfs/s3fileinfo_windows.go
+++ b/vfs/fileinfo_windows.go
@@ -2,6 +2,6 @@ package vfs
import "syscall"
-func (fi S3FileInfo) getFileInfoSys() interface{} {
+func (fi FileInfo) getFileInfoSys() interface{} {
return syscall.Win32FileAttributeData{}
}
diff --git a/vfs/gcsfs.go b/vfs/gcsfs.go
new file mode 100644
index 00000000..3e37700a
--- /dev/null
+++ b/vfs/gcsfs.go
@@ -0,0 +1,511 @@
+package vfs
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "os"
+ "path"
+ "strings"
+ "time"
+
+ "cloud.google.com/go/storage"
+ "github.com/drakkan/sftpgo/logger"
+ "github.com/drakkan/sftpgo/metrics"
+ "github.com/eikenb/pipeat"
+ "google.golang.org/api/googleapi"
+ "google.golang.org/api/iterator"
+ "google.golang.org/api/option"
+)
+
+var (
+ // we cannot use attrs selection until this bug is fixed:
+ //
+ // https://github.com/googleapis/google-cloud-go/issues/1763
+ //
+ gcsDefaultFieldsSelection = []string{"Name", "Size", "Deleted", "Updated"}
+)
+
+// GCSFsConfig defines the configuration for Google Cloud Storage based filesystem
+type GCSFsConfig struct {
+ Bucket string `json:"bucket,omitempty"`
+ // KeyPrefix is similar to a chroot directory for local filesystem.
+ // If specified the SFTP user will only see objects that starts with
+ // this prefix and so you can restrict access to a specific virtual
+ // folder. The prefix, if not empty, must not start with "/" and must
+ // end with "/".
+ // If empty the whole bucket contents will be available
+ KeyPrefix string `json:"key_prefix,omitempty"`
+ CredentialFile string `json:"-"`
+ Credentials string `json:"credentials,omitempty"`
+ StorageClass string `json:"storage_class,omitempty"`
+}
+
+// GCSFs is a Fs implementation for Google Cloud Storage.
+type GCSFs struct {
+ connectionID string
+ localTempDir string
+ config GCSFsConfig
+ svc *storage.Client
+ ctxTimeout time.Duration
+ ctxLongTimeout time.Duration
+}
+
+// NewGCSFs returns an GCSFs object that allows to interact with Google Cloud Storage
+func NewGCSFs(connectionID, localTempDir string, config GCSFsConfig) (Fs, error) {
+ var err error
+ fs := GCSFs{
+ connectionID: connectionID,
+ localTempDir: localTempDir,
+ config: config,
+ ctxTimeout: 30 * time.Second,
+ ctxLongTimeout: 300 * time.Second,
+ }
+ if err = ValidateGCSFsConfig(&fs.config, fs.config.CredentialFile); err != nil {
+ return fs, err
+ }
+ ctx := context.Background()
+ fs.svc, err = storage.NewClient(ctx, option.WithCredentialsFile(fs.config.CredentialFile))
+ return fs, err
+}
+
+// Name returns the name for the Fs implementation
+func (fs GCSFs) Name() string {
+ return fmt.Sprintf("GCSFs bucket: %#v", fs.config.Bucket)
+}
+
+// ConnectionID returns the SSH connection ID associated to this Fs implementation
+func (fs GCSFs) ConnectionID() string {
+ return fs.connectionID
+}
+
+// Stat returns a FileInfo describing the named file
+func (fs GCSFs) Stat(name string) (os.FileInfo, error) {
+ var result FileInfo
+ var err error
+ if len(name) == 0 || name == "." {
+ err := fs.checkIfBucketExists()
+ if err != nil {
+ return result, err
+ }
+ return NewFileInfo(name, true, 0, time.Time{}), nil
+ }
+ if fs.config.KeyPrefix == name+"/" {
+ return NewFileInfo(name, true, 0, time.Time{}), nil
+ }
+ prefix := fs.getPrefixForStat(name)
+ query := &storage.Query{Prefix: prefix, Delimiter: "/"}
+ /*err = query.SetAttrSelection(gcsDefaultFieldsSelection)
+ if err != nil {
+ return result, err
+ }*/
+ ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
+ defer cancelFn()
+ bkt := fs.svc.Bucket(fs.config.Bucket)
+ it := bkt.Objects(ctx, query)
+ for {
+ attrs, err := it.Next()
+ if err == iterator.Done {
+ break
+ }
+ if err != nil {
+ metrics.GCSListObjectsCompleted(err)
+ return result, err
+ }
+ if len(attrs.Prefix) > 0 {
+ if fs.isEqual(attrs.Prefix, name) {
+ result = NewFileInfo(name, true, 0, time.Time{})
+ }
+ } else {
+ if !attrs.Deleted.IsZero() {
+ continue
+ }
+ if fs.isEqual(attrs.Name, name) {
+ isDir := strings.HasSuffix(attrs.Name, "/")
+ result = NewFileInfo(name, isDir, attrs.Size, attrs.Updated)
+ }
+ }
+ }
+ metrics.GCSListObjectsCompleted(nil)
+ if len(result.Name()) == 0 {
+ err = errors.New("404 no such file or directory")
+ }
+ return result, err
+}
+
+// Lstat returns a FileInfo describing the named file
+func (fs GCSFs) Lstat(name string) (os.FileInfo, error) {
+ return fs.Stat(name)
+}
+
+// Open opens the named file for reading
+func (fs GCSFs) Open(name string) (*os.File, *pipeat.PipeReaderAt, func(), error) {
+ r, w, err := pipeat.AsyncWriterPipeInDir(fs.localTempDir)
+ if err != nil {
+ return nil, nil, nil, err
+ }
+ bkt := fs.svc.Bucket(fs.config.Bucket)
+ obj := bkt.Object(name)
+ ctx, cancelFn := context.WithCancel(context.Background())
+ objectReader, err := obj.NewReader(ctx)
+ if err != nil {
+ r.Close()
+ w.Close()
+ cancelFn()
+ return nil, nil, nil, err
+ }
+ go func() {
+ defer cancelFn()
+ defer objectReader.Close()
+ n, err := io.Copy(w, objectReader)
+ w.CloseWithError(err)
+ fsLog(fs, logger.LevelDebug, "download completed, path: %#v size: %v, err: %v", name, n, err)
+ metrics.GCSTransferCompleted(n, 1, err)
+ }()
+ return nil, r, cancelFn, nil
+}
+
+// Create creates or opens the named file for writing
+func (fs GCSFs) Create(name string, flag int) (*os.File, *pipeat.PipeWriterAt, func(), error) {
+ r, w, err := pipeat.PipeInDir(fs.localTempDir)
+ if err != nil {
+ return nil, nil, nil, err
+ }
+ bkt := fs.svc.Bucket(fs.config.Bucket)
+ obj := bkt.Object(name)
+ ctx, cancelFn := context.WithCancel(context.Background())
+ objectWriter := obj.NewWriter(ctx)
+ if len(fs.config.StorageClass) > 0 {
+ objectWriter.ObjectAttrs.StorageClass = fs.config.StorageClass
+ }
+ go func() {
+ defer cancelFn()
+ defer objectWriter.Close()
+ n, err := io.Copy(objectWriter, r)
+ r.CloseWithError(err)
+ fsLog(fs, logger.LevelDebug, "upload completed, path: %#v, readed bytes: %v, err: %v", name, n, err)
+ metrics.GCSTransferCompleted(n, 0, err)
+ }()
+ return nil, w, cancelFn, nil
+}
+
+// Rename renames (moves) source to target.
+// We don't support renaming non empty directories since we should
+// rename all the contents too and this could take long time: think
+// about directories with thousands of files, for each file we should
+// execute a CopyObject call.
+func (fs GCSFs) Rename(source, target string) error {
+ if source == target {
+ return nil
+ }
+ fi, err := fs.Stat(source)
+ if err != nil {
+ return err
+ }
+ if fi.IsDir() {
+ contents, err := fs.ReadDir(source)
+ if err != nil {
+ return err
+ }
+ if len(contents) > 0 {
+ return fmt.Errorf("Cannot rename non empty directory: %#v", source)
+ }
+ if !strings.HasSuffix(source, "/") {
+ source += "/"
+ }
+ if !strings.HasSuffix(target, "/") {
+ target += "/"
+ }
+ }
+ src := fs.svc.Bucket(fs.config.Bucket).Object(source)
+ dst := fs.svc.Bucket(fs.config.Bucket).Object(target)
+ ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
+ defer cancelFn()
+ copier := dst.CopierFrom(src)
+ if len(fs.config.StorageClass) > 0 {
+ copier.StorageClass = fs.config.StorageClass
+ }
+ _, err = copier.Run(ctx)
+ metrics.GCSCopyObjectCompleted(err)
+ if err != nil {
+ return err
+ }
+ return fs.Remove(source, fi.IsDir())
+}
+
+// Remove removes the named file or (empty) directory.
+func (fs GCSFs) Remove(name string, isDir bool) error {
+ if isDir {
+ contents, err := fs.ReadDir(name)
+ if err != nil {
+ return err
+ }
+ if len(contents) > 0 {
+ return fmt.Errorf("Cannot remove non empty directory: %#v", name)
+ }
+ if !strings.HasSuffix(name, "/") {
+ name += "/"
+ }
+ }
+ ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
+ defer cancelFn()
+ err := fs.svc.Bucket(fs.config.Bucket).Object(name).Delete(ctx)
+ metrics.GCSDeleteObjectCompleted(err)
+ return err
+}
+
+// Mkdir creates a new directory with the specified name and default permissions
+func (fs GCSFs) Mkdir(name string) error {
+ _, err := fs.Stat(name)
+ if !fs.IsNotExist(err) {
+ return err
+ }
+ if !strings.HasSuffix(name, "/") {
+ name += "/"
+ }
+ _, w, _, err := fs.Create(name, 0)
+ if err != nil {
+ return err
+ }
+ return w.Close()
+}
+
+// Symlink creates source as a symbolic link to target.
+func (GCSFs) Symlink(source, target string) error {
+ return errors.New("403 symlinks are not supported")
+}
+
+// Chown changes the numeric uid and gid of the named file.
+// Silently ignored.
+func (GCSFs) Chown(name string, uid int, gid int) error {
+ return nil
+}
+
+// Chmod changes the mode of the named file to mode.
+// Silently ignored.
+func (GCSFs) Chmod(name string, mode os.FileMode) error {
+ return nil
+}
+
+// Chtimes changes the access and modification times of the named file.
+// Silently ignored.
+func (GCSFs) Chtimes(name string, atime, mtime time.Time) error {
+ return errors.New("403 chtimes is not supported")
+}
+
+// ReadDir reads the directory named by dirname and returns
+// a list of directory entries.
+func (fs GCSFs) ReadDir(dirname string) ([]os.FileInfo, error) {
+ var result []os.FileInfo
+ // dirname deve essere già cleaned
+ prefix := ""
+ if len(dirname) > 0 && dirname != "." {
+ prefix = strings.TrimPrefix(dirname, "/")
+ if !strings.HasSuffix(prefix, "/") {
+ prefix += "/"
+ }
+ }
+ query := &storage.Query{Prefix: prefix, Delimiter: "/"}
+ /*err := query.SetAttrSelection(gcsDefaultFieldsSelection)
+ if err != nil {
+ return result, err
+ }*/
+ ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
+ defer cancelFn()
+ bkt := fs.svc.Bucket(fs.config.Bucket)
+ it := bkt.Objects(ctx, query)
+ for {
+ attrs, err := it.Next()
+ if err == iterator.Done {
+ break
+ }
+ if err != nil {
+ metrics.GCSListObjectsCompleted(err)
+ return result, err
+ }
+ if len(attrs.Prefix) > 0 {
+ name, _ := fs.resolve(attrs.Prefix, prefix)
+ result = append(result, NewFileInfo(name, true, 0, time.Time{}))
+ } else {
+ name, isDir := fs.resolve(attrs.Name, prefix)
+ if len(name) == 0 {
+ continue
+ }
+ if !attrs.Deleted.IsZero() {
+ continue
+ }
+ result = append(result, NewFileInfo(name, isDir, attrs.Size, attrs.Updated))
+ }
+ }
+ metrics.GCSListObjectsCompleted(nil)
+ return result, nil
+}
+
+// IsUploadResumeSupported returns true if upload resume is supported.
+// SFTP Resume is not supported on S3
+func (GCSFs) IsUploadResumeSupported() bool {
+ return false
+}
+
+// IsAtomicUploadSupported returns true if atomic upload is supported.
+// S3 uploads are already atomic, we don't need to upload to a temporary
+// file
+func (GCSFs) IsAtomicUploadSupported() bool {
+ return false
+}
+
+// IsNotExist returns a boolean indicating whether the error is known to
+// report that a file or directory does not exist
+func (GCSFs) IsNotExist(err error) bool {
+ if err == nil {
+ return false
+ }
+ if err == storage.ErrObjectNotExist || err == storage.ErrBucketNotExist {
+ return true
+ }
+ if e, ok := err.(*googleapi.Error); ok {
+ if e.Code == http.StatusNotFound {
+ return true
+ }
+ }
+ return strings.Contains(err.Error(), "404")
+}
+
+// IsPermission returns a boolean indicating whether the error is known to
+// report that permission is denied.
+func (GCSFs) IsPermission(err error) bool {
+ if err == nil {
+ return false
+ }
+ if e, ok := err.(*googleapi.Error); ok {
+ if e.Code == http.StatusForbidden || e.Code == http.StatusUnauthorized {
+ return true
+ }
+ }
+ return strings.Contains(err.Error(), "403")
+}
+
+// CheckRootPath creates the specified root directory if it does not exists
+func (fs GCSFs) CheckRootPath(username string, uid int, gid int) bool {
+ // we need a local directory for temporary files
+ osFs := NewOsFs(fs.ConnectionID(), fs.localTempDir)
+ osFs.CheckRootPath(username, uid, gid)
+ return fs.checkIfBucketExists() != nil
+}
+
+// ScanRootDirContents returns the number of files contained in the bucket,
+// and their size
+func (fs GCSFs) ScanRootDirContents() (int, int64, error) {
+ numFiles := 0
+ size := int64(0)
+ query := &storage.Query{Prefix: fs.config.KeyPrefix}
+ err := query.SetAttrSelection(gcsDefaultFieldsSelection)
+ if err != nil {
+ return numFiles, size, err
+ }
+ ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxLongTimeout))
+ defer cancelFn()
+ bkt := fs.svc.Bucket(fs.config.Bucket)
+ it := bkt.Objects(ctx, query)
+ for {
+ attrs, err := it.Next()
+ if err == iterator.Done {
+ break
+ }
+ if err != nil {
+ metrics.GCSListObjectsCompleted(err)
+ return numFiles, size, err
+ }
+ if !attrs.Deleted.IsZero() {
+ continue
+ }
+ numFiles++
+ size += attrs.Size
+ }
+ metrics.GCSListObjectsCompleted(nil)
+ return numFiles, size, err
+}
+
+// GetAtomicUploadPath returns the path to use for an atomic upload.
+// S3 uploads are already atomic, we never call this method for S3
+func (GCSFs) GetAtomicUploadPath(name string) string {
+ return ""
+}
+
+// GetRelativePath returns the path for a file relative to the user's home dir.
+// This is the path as seen by SFTP users
+func (fs GCSFs) GetRelativePath(name string) string {
+ rel := path.Clean(name)
+ if rel == "." {
+ rel = ""
+ }
+ if !path.IsAbs(rel) {
+ rel = "/" + rel
+ }
+ if len(fs.config.KeyPrefix) > 0 {
+ if !strings.HasPrefix(rel, "/"+fs.config.KeyPrefix) {
+ rel = "/"
+ }
+ rel = path.Clean("/" + strings.TrimPrefix(rel, "/"+fs.config.KeyPrefix))
+ }
+ return rel
+}
+
+// Join joins any number of path elements into a single path
+func (GCSFs) Join(elem ...string) string {
+ return strings.TrimPrefix(path.Join(elem...), "/")
+}
+
+// ResolvePath returns the matching filesystem path for the specified sftp path
+func (fs GCSFs) ResolvePath(sftpPath string) (string, error) {
+ if !path.IsAbs(sftpPath) {
+ sftpPath = path.Clean("/" + sftpPath)
+ }
+ return fs.Join(fs.config.KeyPrefix, strings.TrimPrefix(sftpPath, "/")), nil
+}
+
+func (fs *GCSFs) resolve(name string, prefix string) (string, bool) {
+ result := strings.TrimPrefix(name, prefix)
+ isDir := strings.HasSuffix(result, "/")
+ if isDir {
+ result = strings.TrimSuffix(result, "/")
+ }
+ return result, isDir
+}
+
+func (fs *GCSFs) isEqual(key string, sftpName string) bool {
+ if key == sftpName {
+ return true
+ }
+ if key == sftpName+"/" {
+ return true
+ }
+ if key+"/" == sftpName {
+ return true
+ }
+ return false
+}
+
+func (fs *GCSFs) checkIfBucketExists() error {
+ ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
+ defer cancelFn()
+ bkt := fs.svc.Bucket(fs.config.Bucket)
+ _, err := bkt.Attrs(ctx)
+ metrics.GCSHeadBucketCompleted(err)
+ return err
+}
+
+func (fs *GCSFs) getPrefixForStat(name string) string {
+ prefix := path.Dir(name)
+ if prefix == "/" || prefix == "." || len(prefix) == 0 {
+ prefix = ""
+ } else {
+ prefix = strings.TrimPrefix(prefix, "/")
+ if !strings.HasSuffix(prefix, "/") {
+ prefix += "/"
+ }
+ }
+ return prefix
+}
diff --git a/vfs/s3fs.go b/vfs/s3fs.go
index 3a350bcc..56ad447d 100644
--- a/vfs/s3fs.go
+++ b/vfs/s3fs.go
@@ -21,15 +21,15 @@ import (
"github.com/eikenb/pipeat"
)
-// S3FsConfig defines the configuration for S3fs
+// S3FsConfig defines the configuration for S3 based filesystem
type S3FsConfig struct {
Bucket string `json:"bucket,omitempty"`
// KeyPrefix is similar to a chroot directory for local filesystem.
- // If specified the SFTP user will only see contents that starts with
+ // If specified the SFTP user will only see objects that starts with
// this prefix and so you can restrict access to a specific virtual
// folder. The prefix, if not empty, must not start with "/" and must
// end with "/".
- //If empty the whole bucket contents will be available
+ // If empty the whole bucket contents will be available
KeyPrefix string `json:"key_prefix,omitempty"`
Region string `json:"region,omitempty"`
AccessKey string `json:"access_key,omitempty"`
@@ -70,7 +70,6 @@ func NewS3Fs(connectionID, localTempDir string, config S3FsConfig) (Fs, error) {
Region: aws.String(fs.config.Region),
Credentials: credentials.NewStaticCredentials(fs.config.AccessKey, fs.config.AccessSecret, ""),
}
- //config.WithLogLevel(aws.LogDebugWithHTTPBody)
if len(fs.config.Endpoint) > 0 {
awsConfig.Endpoint = aws.String(fs.config.Endpoint)
awsConfig.S3ForcePathStyle = aws.Bool(true)
@@ -95,16 +94,16 @@ func (fs S3Fs) ConnectionID() string {
// Stat returns a FileInfo describing the named file
func (fs S3Fs) Stat(name string) (os.FileInfo, error) {
- var result S3FileInfo
+ var result FileInfo
if name == "/" || name == "." {
err := fs.checkIfBucketExists()
if err != nil {
return result, err
}
- return NewS3FileInfo(name, true, 0, time.Time{}), nil
+ return NewFileInfo(name, true, 0, time.Time{}), nil
}
if "/"+fs.config.KeyPrefix == name+"/" {
- return NewS3FileInfo(name, true, 0, time.Time{}), nil
+ return NewFileInfo(name, true, 0, time.Time{}), nil
}
prefix := path.Dir(name)
if prefix == "/" || prefix == "." {
@@ -124,7 +123,7 @@ func (fs S3Fs) Stat(name string) (os.FileInfo, error) {
}, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
for _, p := range page.CommonPrefixes {
if fs.isEqual(p.Prefix, name) {
- result = NewS3FileInfo(name, true, 0, time.Time{})
+ result = NewFileInfo(name, true, 0, time.Time{})
return false
}
}
@@ -133,7 +132,7 @@ func (fs S3Fs) Stat(name string) (os.FileInfo, error) {
objectSize := *fileObject.Size
objectModTime := *fileObject.LastModified
isDir := strings.HasSuffix(*fileObject.Key, "/")
- result = NewS3FileInfo(name, isDir, objectSize, objectModTime)
+ result = NewFileInfo(name, isDir, objectSize, objectModTime)
return false
}
}
@@ -325,7 +324,7 @@ func (fs S3Fs) ReadDir(dirname string) ([]os.FileInfo, error) {
}, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
for _, p := range page.CommonPrefixes {
name, isDir := fs.resolve(p.Prefix, prefix)
- result = append(result, NewS3FileInfo(name, isDir, 0, time.Time{}))
+ result = append(result, NewFileInfo(name, isDir, 0, time.Time{}))
}
for _, fileObject := range page.Contents {
objectSize := *fileObject.Size
@@ -334,7 +333,7 @@ func (fs S3Fs) ReadDir(dirname string) ([]os.FileInfo, error) {
if len(name) == 0 {
continue
}
- result = append(result, NewS3FileInfo(name, isDir, objectSize, objectModTime))
+ result = append(result, NewFileInfo(name, isDir, objectSize, objectModTime))
}
return true
})
@@ -394,23 +393,7 @@ func (fs S3Fs) CheckRootPath(username string, uid int, gid int) bool {
// we need a local directory for temporary files
osFs := NewOsFs(fs.ConnectionID(), fs.localTempDir)
osFs.CheckRootPath(username, uid, gid)
- err := fs.checkIfBucketExists()
- if err == nil {
- return true
- }
- if !fs.IsNotExist(err) {
- return false
- }
- ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
- defer cancelFn()
- input := &s3.CreateBucketInput{
- Bucket: aws.String(fs.config.Bucket),
- }
- _, err = fs.svc.CreateBucketWithContext(ctx, input)
- fsLog(fs, logger.LevelDebug, "bucket %#v for user %#v does not exists, try to create, error: %v",
- fs.config.Bucket, username, err)
- metrics.S3CreateBucketCompleted(err)
- return err == nil
+ return fs.checkIfBucketExists() != nil
}
// ScanRootDirContents returns the number of files contained in the bucket,
diff --git a/vfs/vfs.go b/vfs/vfs.go
index 46d25f82..4952619d 100644
--- a/vfs/vfs.go
+++ b/vfs/vfs.go
@@ -3,6 +3,7 @@ package vfs
import (
"errors"
+ "fmt"
"os"
"path"
"runtime"
@@ -14,7 +15,7 @@ import (
"github.com/pkg/sftp"
)
-// Fs defines the interface for filesystems backends
+// Fs defines the interface for filesystem backends
type Fs interface {
Name() string
ConnectionID() string
@@ -94,6 +95,32 @@ func ValidateS3FsConfig(config *S3FsConfig) error {
return nil
}
+// ValidateGCSFsConfig returns nil if the specified GCS config is valid, otherwise an error
+func ValidateGCSFsConfig(config *GCSFsConfig, credentialsFilePath string) error {
+ if len(config.Bucket) == 0 {
+ return errors.New("bucket cannot be empty")
+ }
+ if len(config.KeyPrefix) > 0 {
+ if strings.HasPrefix(config.KeyPrefix, "/") {
+ return errors.New("key_prefix cannot start with /")
+ }
+ config.KeyPrefix = path.Clean(config.KeyPrefix)
+ if !strings.HasSuffix(config.KeyPrefix, "/") {
+ config.KeyPrefix += "/"
+ }
+ }
+ if len(config.Credentials) == 0 {
+ fi, err := os.Stat(credentialsFilePath)
+ if err != nil {
+ return fmt.Errorf("invalid credentials %v", err)
+ }
+ if fi.Size() == 0 {
+ return errors.New("credentials cannot be empty")
+ }
+ }
+ return nil
+}
+
// SetPathPermissions calls fs.Chown.
// It does nothing for local filesystem on windows
func SetPathPermissions(fs Fs, path string, uid int, gid int) {