From eec9c449d4c7569d87d2f537b420425de4520721 Mon Sep 17 00:00:00 2001 From: Nicola Murino Date: Wed, 24 Jan 2024 19:59:50 +0100 Subject: [PATCH] vfs: make PipeReader an interface Signed-off-by: Nicola Murino --- internal/ftpd/transfer.go | 2 +- internal/httpd/file.go | 2 +- internal/sftpd/sftpd_test.go | 4 ++-- internal/sftpd/transfer.go | 2 +- internal/vfs/azblobfs.go | 4 ++-- internal/vfs/cryptfs.go | 2 +- internal/vfs/gcsfs.go | 2 +- internal/vfs/httpfs.go | 2 +- internal/vfs/osfs.go | 2 +- internal/vfs/s3fs.go | 2 +- internal/vfs/sftpfs.go | 2 +- internal/vfs/vfs.go | 31 ++++++++++++++++++++++--------- internal/webdavd/internal_test.go | 2 +- 13 files changed, 36 insertions(+), 23 deletions(-) diff --git a/internal/ftpd/transfer.go b/internal/ftpd/transfer.go index 11399cd7..ed0ce660 100644 --- a/internal/ftpd/transfer.go +++ b/internal/ftpd/transfer.go @@ -32,7 +32,7 @@ type transfer struct { expectedOffset int64 } -func newTransfer(baseTransfer *common.BaseTransfer, pipeWriter vfs.PipeWriter, pipeReader *vfs.PipeReader, +func newTransfer(baseTransfer *common.BaseTransfer, pipeWriter vfs.PipeWriter, pipeReader vfs.PipeReader, expectedOffset int64) *transfer { var writer io.WriteCloser var reader io.ReadCloser diff --git a/internal/httpd/file.go b/internal/httpd/file.go index 3d458ffb..5bc4627b 100644 --- a/internal/httpd/file.go +++ b/internal/httpd/file.go @@ -28,7 +28,7 @@ type httpdFile struct { isFinished bool } -func newHTTPDFile(baseTransfer *common.BaseTransfer, pipeWriter vfs.PipeWriter, pipeReader *vfs.PipeReader) *httpdFile { +func newHTTPDFile(baseTransfer *common.BaseTransfer, pipeWriter vfs.PipeWriter, pipeReader vfs.PipeReader) *httpdFile { var writer io.WriteCloser var reader io.ReadCloser if baseTransfer.File != nil { diff --git a/internal/sftpd/sftpd_test.go b/internal/sftpd/sftpd_test.go index abdb6009..a5813322 100644 --- a/internal/sftpd/sftpd_test.go +++ b/internal/sftpd/sftpd_test.go @@ -8939,8 +8939,8 @@ func TestStatVFSCloudBackend(t *testing.T) { u := getTestUser(usePubKey) u.FsConfig.Provider = sdk.AzureBlobFilesystemProvider u.FsConfig.AzBlobConfig.SASURL = kms.NewPlainSecret("https://myaccount.blob.core.windows.net/sasurl") - user, _, err := httpdtest.AddUser(u, http.StatusCreated) - assert.NoError(t, err) + user, resp, err := httpdtest.AddUser(u, http.StatusCreated) + assert.NoError(t, err, string(resp)) conn, client, err := getSftpClient(user, usePubKey) if assert.NoError(t, err) { defer conn.Close() diff --git a/internal/sftpd/transfer.go b/internal/sftpd/transfer.go index 4b2c4cb6..d3acc202 100644 --- a/internal/sftpd/transfer.go +++ b/internal/sftpd/transfer.go @@ -58,7 +58,7 @@ type transfer struct { isFinished bool } -func newTransfer(baseTransfer *common.BaseTransfer, pipeWriter vfs.PipeWriter, pipeReader *vfs.PipeReader, +func newTransfer(baseTransfer *common.BaseTransfer, pipeWriter vfs.PipeWriter, pipeReader vfs.PipeReader, errForRead error) *transfer { var writer writerAtCloser var reader readerAtCloser diff --git a/internal/vfs/azblobfs.go b/internal/vfs/azblobfs.go index f5c9f006..3a23ccce 100644 --- a/internal/vfs/azblobfs.go +++ b/internal/vfs/azblobfs.go @@ -202,7 +202,7 @@ func (fs *AzureBlobFs) Lstat(name string) (os.FileInfo, error) { } // Open opens the named file for reading -func (fs *AzureBlobFs) Open(name string, offset int64) (File, *PipeReader, func(), error) { +func (fs *AzureBlobFs) Open(name string, offset int64) (File, PipeReader, func(), error) { r, w, err := pipeat.PipeInDir(fs.localTempDir) if err != nil { return nil, nil, nil, err @@ -991,7 +991,7 @@ func (fs *AzureBlobFs) downloadPart(ctx context.Context, blockBlob *blockblob.Cl } func (fs *AzureBlobFs) handleMultipartDownload(ctx context.Context, blockBlob *blockblob.Client, - offset int64, writer io.WriterAt, pipeReader *PipeReader, + offset int64, writer io.WriterAt, pipeReader PipeReader, ) error { props, err := blockBlob.GetProperties(ctx, &blob.GetPropertiesOptions{}) metric.AZHeadObjectCompleted(err) diff --git a/internal/vfs/cryptfs.go b/internal/vfs/cryptfs.go index 0d1ac03e..d8e82cc5 100644 --- a/internal/vfs/cryptfs.go +++ b/internal/vfs/cryptfs.go @@ -79,7 +79,7 @@ func (fs *CryptFs) Name() string { } // Open opens the named file for reading -func (fs *CryptFs) Open(name string, offset int64) (File, *PipeReader, func(), error) { +func (fs *CryptFs) Open(name string, offset int64) (File, PipeReader, func(), error) { f, key, err := fs.getFileAndEncryptionKey(name) if err != nil { return nil, nil, nil, err diff --git a/internal/vfs/gcsfs.go b/internal/vfs/gcsfs.go index dafec98e..c1803875 100644 --- a/internal/vfs/gcsfs.go +++ b/internal/vfs/gcsfs.go @@ -126,7 +126,7 @@ func (fs *GCSFs) Lstat(name string) (os.FileInfo, error) { } // Open opens the named file for reading -func (fs *GCSFs) Open(name string, offset int64) (File, *PipeReader, func(), error) { +func (fs *GCSFs) Open(name string, offset int64) (File, PipeReader, func(), error) { r, w, err := pipeat.PipeInDir(fs.localTempDir) if err != nil { return nil, nil, nil, err diff --git a/internal/vfs/httpfs.go b/internal/vfs/httpfs.go index 04399b7a..67d2c915 100644 --- a/internal/vfs/httpfs.go +++ b/internal/vfs/httpfs.go @@ -316,7 +316,7 @@ func (fs *HTTPFs) Lstat(name string) (os.FileInfo, error) { } // Open opens the named file for reading -func (fs *HTTPFs) Open(name string, offset int64) (File, *PipeReader, func(), error) { +func (fs *HTTPFs) Open(name string, offset int64) (File, PipeReader, func(), error) { r, w, err := pipeat.PipeInDir(fs.localTempDir) if err != nil { return nil, nil, nil, err diff --git a/internal/vfs/osfs.go b/internal/vfs/osfs.go index cf742ef9..7f8de472 100644 --- a/internal/vfs/osfs.go +++ b/internal/vfs/osfs.go @@ -101,7 +101,7 @@ func (fs *OsFs) Lstat(name string) (os.FileInfo, error) { } // Open opens the named file for reading -func (fs *OsFs) Open(name string, offset int64) (File, *PipeReader, func(), error) { +func (fs *OsFs) Open(name string, offset int64) (File, PipeReader, func(), error) { f, err := os.Open(name) if err != nil { return nil, nil, nil, err diff --git a/internal/vfs/s3fs.go b/internal/vfs/s3fs.go index e21f6f46..9b2bdcf9 100644 --- a/internal/vfs/s3fs.go +++ b/internal/vfs/s3fs.go @@ -196,7 +196,7 @@ func (fs *S3Fs) Lstat(name string) (os.FileInfo, error) { } // Open opens the named file for reading -func (fs *S3Fs) Open(name string, offset int64) (File, *PipeReader, func(), error) { +func (fs *S3Fs) Open(name string, offset int64) (File, PipeReader, func(), error) { r, w, err := pipeat.PipeInDir(fs.localTempDir) if err != nil { return nil, nil, nil, err diff --git a/internal/vfs/sftpfs.go b/internal/vfs/sftpfs.go index 8c2b4ae3..929ef7d6 100644 --- a/internal/vfs/sftpfs.go +++ b/internal/vfs/sftpfs.go @@ -346,7 +346,7 @@ func (fs *SFTPFs) Lstat(name string) (os.FileInfo, error) { } // Open opens the named file for reading -func (fs *SFTPFs) Open(name string, offset int64) (File, *PipeReader, func(), error) { +func (fs *SFTPFs) Open(name string, offset int64) (File, PipeReader, func(), error) { client, err := fs.conn.getClient() if err != nil { return nil, nil, nil, err diff --git a/internal/vfs/vfs.go b/internal/vfs/vfs.go index 683bcfef..473746ee 100644 --- a/internal/vfs/vfs.go +++ b/internal/vfs/vfs.go @@ -115,7 +115,7 @@ type Fs interface { ConnectionID() string Stat(name string) (os.FileInfo, error) Lstat(name string) (os.FileInfo, error) - Open(name string, offset int64) (File, *PipeReader, func(), error) + Open(name string, offset int64) (File, PipeReader, func(), error) Create(name string, flag, checks int) (File, PipeWriter, func(), error) Rename(source, target string) (int, int64, error) Remove(name string, isDir bool) error @@ -189,6 +189,16 @@ type PipeWriter interface { GetWrittenBytes() int64 } +// PipeReader defines an interface representing a SFTPGo pipe writer +type PipeReader interface { + io.Reader + io.ReaderAt + io.Closer + setMetadata(value map[string]string) + setMetadataFromPointerVal(value map[string]*string) + Metadata() map[string]string +} + // Metadater defines an interface to implement to return metadata for a file type Metadater interface { Metadata() map[string]string @@ -628,7 +638,10 @@ func (c *AzBlobFsConfig) ValidateAndEncryptCredentials(additionalData string) er func (c *AzBlobFsConfig) checkCredentials() error { if c.SASURL.IsPlain() { _, err := url.Parse(c.SASURL.GetPayload()) - return util.NewI18nError(err, util.I18nErrorSASURLInvalid) + if err != nil { + return util.NewI18nError(err, util.I18nErrorSASURLInvalid) + } + return nil } if c.SASURL.IsEncrypted() && !c.SASURL.IsValid() { return errors.New("invalid encrypted sas_url") @@ -851,27 +864,27 @@ func (p *pipeWriterAtOffset) Write(buf []byte) (int, error) { } // NewPipeReader initializes a new PipeReader -func NewPipeReader(r *pipeat.PipeReaderAt) *PipeReader { - return &PipeReader{ +func NewPipeReader(r *pipeat.PipeReaderAt) PipeReader { + return &pipeReader{ PipeReaderAt: r, } } -// PipeReader defines a wrapper for pipeat.PipeReaderAt. -type PipeReader struct { +// pipeReader defines a wrapper for pipeat.PipeReaderAt. +type pipeReader struct { *pipeat.PipeReaderAt mu sync.RWMutex metadata map[string]string } -func (p *PipeReader) setMetadata(value map[string]string) { +func (p *pipeReader) setMetadata(value map[string]string) { p.mu.Lock() defer p.mu.Unlock() p.metadata = value } -func (p *PipeReader) setMetadataFromPointerVal(value map[string]*string) { +func (p *pipeReader) setMetadataFromPointerVal(value map[string]*string) { p.mu.Lock() defer p.mu.Unlock() @@ -890,7 +903,7 @@ func (p *PipeReader) setMetadataFromPointerVal(value map[string]*string) { } // Metadata implements the Metadater interface -func (p *PipeReader) Metadata() map[string]string { +func (p *pipeReader) Metadata() map[string]string { p.mu.RLock() defer p.mu.RUnlock() diff --git a/internal/webdavd/internal_test.go b/internal/webdavd/internal_test.go index 8abfd4eb..9997d644 100644 --- a/internal/webdavd/internal_test.go +++ b/internal/webdavd/internal_test.go @@ -286,7 +286,7 @@ func (fs *MockOsFs) Name() string { } // Open returns nil -func (fs *MockOsFs) Open(name string, offset int64) (vfs.File, *vfs.PipeReader, func(), error) { +func (fs *MockOsFs) Open(name string, offset int64) (vfs.File, vfs.PipeReader, func(), error) { if fs.reader != nil { return nil, vfs.NewPipeReader(fs.reader), nil, nil }