diff --git a/internal/common/connection.go b/internal/common/connection.go index 144ecaaa..5b7a0953 100644 --- a/internal/common/connection.go +++ b/internal/common/connection.go @@ -620,7 +620,7 @@ func (c *BaseConnection) checkCopy(srcInfo, dstInfo os.FileInfo, virtualSource, return nil } -func (c *BaseConnection) copyFile(virtualSourcePath, virtualTargetPath string, srcSize int64) error { +func (c *BaseConnection) copyFile(virtualSourcePath, virtualTargetPath string, srcInfo os.FileInfo) error { if !c.User.HasPerm(dataprovider.PermCopy, virtualSourcePath) || !c.User.HasPerm(dataprovider.PermCopy, virtualTargetPath) { return c.GetPermissionDeniedError() } @@ -638,12 +638,12 @@ func (c *BaseConnection) copyFile(virtualSourcePath, virtualTargetPath string, s return err } startTime := time.Now() - numFiles, sizeDiff, err := copier.CopyFile(fsSourcePath, fsTargetPath, srcSize) + numFiles, sizeDiff, err := copier.CopyFile(fsSourcePath, fsTargetPath, srcInfo) elapsed := time.Since(startTime).Nanoseconds() / 1000000 updateUserQuotaAfterFileWrite(c, virtualTargetPath, numFiles, sizeDiff) logger.CommandLog(copyLogSender, fsSourcePath, fsTargetPath, c.User.Username, "", c.ID, c.protocol, -1, -1, - "", "", "", srcSize, c.localAddr, c.remoteAddr, elapsed) - ExecuteActionNotification(c, operationCopy, fsSourcePath, virtualSourcePath, fsTargetPath, virtualTargetPath, "", srcSize, err, elapsed, nil) //nolint:errcheck + "", "", "", srcInfo.Size(), c.localAddr, c.remoteAddr, elapsed) + ExecuteActionNotification(c, operationCopy, fsSourcePath, virtualSourcePath, fsTargetPath, virtualTargetPath, "", srcInfo.Size(), err, elapsed, nil) //nolint:errcheck return err } } @@ -655,7 +655,7 @@ func (c *BaseConnection) copyFile(virtualSourcePath, virtualTargetPath string, s defer rCancelFn() defer reader.Close() - writer, numFiles, truncatedSize, wCancelFn, err := getFileWriter(c, virtualTargetPath, srcSize) + writer, numFiles, truncatedSize, wCancelFn, err := getFileWriter(c, virtualTargetPath, srcInfo.Size()) if err != nil { return fmt.Errorf("unable to get writer for path %q: %w", virtualTargetPath, err) } @@ -706,7 +706,7 @@ func (c *BaseConnection) doRecursiveCopy(virtualSourcePath, virtualTargetPath st return nil } - return c.copyFile(virtualSourcePath, virtualTargetPath, srcInfo.Size()) + return c.copyFile(virtualSourcePath, virtualTargetPath, srcInfo) } func (c *BaseConnection) recursiveCopyEntries(virtualSourcePath, virtualTargetPath string, entries []os.FileInfo, recursion int) error { diff --git a/internal/vfs/azblobfs.go b/internal/vfs/azblobfs.go index 067d612c..11129167 100644 --- a/internal/vfs/azblobfs.go +++ b/internal/vfs/azblobfs.go @@ -186,7 +186,11 @@ func (fs *AzureBlobFs) Stat(name string) (os.FileInfo, error) { if val := getAzureLastModified(attrs.Metadata); val > 0 { lastModified = util.GetTimeFromMsecSinceEpoch(val) } - return NewFileInfo(name, isDir, util.GetIntFromPointer(attrs.ContentLength), lastModified, false), nil + info := NewFileInfo(name, isDir, util.GetIntFromPointer(attrs.ContentLength), lastModified, false) + if !isDir { + info.setMetadataFromPointerVal(attrs.Metadata) + } + return info, nil } if !fs.IsNotExist(err) { return nil, err @@ -651,9 +655,9 @@ func (fs *AzureBlobFs) ResolvePath(virtualPath string) (string, error) { } // CopyFile implements the FsFileCopier interface -func (fs *AzureBlobFs) CopyFile(source, target string, srcSize int64) (int, int64, error) { +func (fs *AzureBlobFs) CopyFile(source, target string, srcInfo os.FileInfo) (int, int64, error) { numFiles := 1 - sizeDiff := srcSize + sizeDiff := srcInfo.Size() attrs, err := fs.headObject(target) if err == nil { sizeDiff -= util.GetIntFromPointer(attrs.ContentLength) @@ -663,7 +667,7 @@ func (fs *AzureBlobFs) CopyFile(source, target string, srcSize int64) (int, int6 return 0, 0, err } } - if err := fs.copyFileInternal(source, target); err != nil { + if err := fs.copyFileInternal(source, target, srcInfo); err != nil { return 0, 0, err } return numFiles, sizeDiff, nil @@ -746,13 +750,13 @@ func (fs *AzureBlobFs) setConfigDefaults() { } } -func (fs *AzureBlobFs) copyFileInternal(source, target string) error { +func (fs *AzureBlobFs) copyFileInternal(source, target string, srcInfo os.FileInfo) error { ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxLongTimeout)) defer cancelFn() srcBlob := fs.containerClient.NewBlockBlobClient(source) dstBlob := fs.containerClient.NewBlockBlobClient(target) - resp, err := dstBlob.StartCopyFromURL(ctx, srcBlob.URL(), fs.getCopyOptions()) + resp, err := dstBlob.StartCopyFromURL(ctx, srcBlob.URL(), fs.getCopyOptions(srcInfo)) if err != nil { metric.AZCopyObjectCompleted(err) return err @@ -785,11 +789,11 @@ func (fs *AzureBlobFs) copyFileInternal(source, target string) error { return nil } -func (fs *AzureBlobFs) renameInternal(source, target string, fi os.FileInfo, recursion int) (int, int64, error) { +func (fs *AzureBlobFs) renameInternal(source, target string, srcInfo os.FileInfo, recursion int) (int, int64, error) { var numFiles int var filesSize int64 - if fi.IsDir() { + if srcInfo.IsDir() { if renameMode == 0 { hasContents, err := fs.hasContents(source) if err != nil { @@ -811,13 +815,13 @@ func (fs *AzureBlobFs) renameInternal(source, target string, fi os.FileInfo, rec } } } else { - if err := fs.copyFileInternal(source, target); err != nil { + if err := fs.copyFileInternal(source, target, srcInfo); err != nil { return numFiles, filesSize, err } numFiles++ - filesSize += fi.Size() + filesSize += srcInfo.Size() } - err := fs.skipNotExistErr(fs.Remove(source, fi.IsDir())) + err := fs.skipNotExistErr(fs.Remove(source, srcInfo.IsDir())) return numFiles, filesSize, err } @@ -1098,11 +1102,20 @@ func (*AzureBlobFs) readFill(r io.Reader, buf []byte) (n int, err error) { return n, err } -func (fs *AzureBlobFs) getCopyOptions() *blob.StartCopyFromURLOptions { +func (fs *AzureBlobFs) getCopyOptions(srcInfo os.FileInfo) *blob.StartCopyFromURLOptions { copyOptions := &blob.StartCopyFromURLOptions{} if fs.config.AccessTier != "" { copyOptions.Tier = (*blob.AccessTier)(&fs.config.AccessTier) } + metadata := make(map[string]*string) + for k, v := range getMetadata(srcInfo) { + if v != "" { + metadata[k] = to.Ptr(v) + } + } + if len(metadata) > 0 { + copyOptions.Metadata = metadata + } return copyOptions } @@ -1254,6 +1267,7 @@ func (l *azureBlobDirLister) Next(limit int) ([]os.FileInfo, error) { name = strings.TrimPrefix(name, l.prefix) size := int64(0) isDir := false + var metadata map[string]*string modTime := time.Unix(0, 0) if blobItem.Properties != nil { size = util.GetIntFromPointer(blobItem.Properties.ContentLength) @@ -1266,12 +1280,16 @@ func (l *azureBlobDirLister) Next(limit int) ([]os.FileInfo, error) { continue } l.prefixes[name] = true + } else { + metadata = blobItem.Metadata } if val := getAzureLastModified(blobItem.Metadata); val > 0 { modTime = util.GetTimeFromMsecSinceEpoch(val) } } - l.cache = append(l.cache, NewFileInfo(name, isDir, size, modTime, false)) + info := NewFileInfo(name, isDir, size, modTime, false) + info.setMetadataFromPointerVal(metadata) + l.cache = append(l.cache, info) } return l.returnFromCache(limit), nil diff --git a/internal/vfs/fileinfo.go b/internal/vfs/fileinfo.go index a6079443..dd0c153a 100644 --- a/internal/vfs/fileinfo.go +++ b/internal/vfs/fileinfo.go @@ -18,6 +18,8 @@ import ( "os" "path" "time" + + "github.com/drakkan/sftpgo/v2/internal/util" ) // FileInfo implements os.FileInfo for a Cloud Storage file. @@ -26,6 +28,7 @@ type FileInfo struct { sizeInBytes int64 modTime time.Time mode os.FileMode + metadata map[string]string } // NewFileInfo creates file info. @@ -79,5 +82,33 @@ func (fi *FileInfo) SetMode(mode os.FileMode) { // Sys provides the underlying data source (can return nil) func (fi *FileInfo) Sys() any { + return fi.metadata +} + +func (fi *FileInfo) setMetadata(value map[string]string) { + fi.metadata = value +} + +func (fi *FileInfo) setMetadataFromPointerVal(value map[string]*string) { + if len(value) == 0 { + fi.metadata = nil + return + } + + fi.metadata = map[string]string{} + for k, v := range value { + val := util.GetStringFromPointer(v) + if val != "" { + fi.metadata[k] = val + } + } +} + +func getMetadata(fi os.FileInfo) map[string]string { + if val, ok := fi.Sys().(map[string]string); ok { + if len(val) > 0 { + return val + } + } return nil } diff --git a/internal/vfs/gcsfs.go b/internal/vfs/gcsfs.go index 613e67ff..06b55323 100644 --- a/internal/vfs/gcsfs.go +++ b/internal/vfs/gcsfs.go @@ -636,9 +636,9 @@ func (fs *GCSFs) ResolvePath(virtualPath string) (string, error) { } // CopyFile implements the FsFileCopier interface -func (fs *GCSFs) CopyFile(source, target string, srcSize int64) (int, int64, error) { +func (fs *GCSFs) CopyFile(source, target string, srcInfo os.FileInfo) (int, int64, error) { numFiles := 1 - sizeDiff := srcSize + sizeDiff := srcInfo.Size() var conditions *storage.Conditions attrs, err := fs.headObject(target) if err == nil { @@ -651,7 +651,7 @@ func (fs *GCSFs) CopyFile(source, target string, srcSize int64) (int, int64, err } conditions = &storage.Conditions{DoesNotExist: true} } - if err := fs.copyFileInternal(source, target, conditions); err != nil { + if err := fs.copyFileInternal(source, target, conditions, srcInfo); err != nil { return 0, 0, err } return numFiles, sizeDiff, nil @@ -679,7 +679,11 @@ func (fs *GCSFs) getObjectStat(name string) (os.FileInfo, error) { objectModTime = util.GetTimeFromMsecSinceEpoch(val) } isDir := attrs.ContentType == dirMimeType || strings.HasSuffix(attrs.Name, "/") - return NewFileInfo(name, isDir, objSize, objectModTime, false), nil + info := NewFileInfo(name, isDir, objSize, objectModTime, false) + if !isDir { + info.setMetadata(attrs.Metadata) + } + return info, nil } if !fs.IsNotExist(err) { return nil, err @@ -749,7 +753,7 @@ func (fs *GCSFs) composeObjects(ctx context.Context, dst, partialObject *storage return err } -func (fs *GCSFs) copyFileInternal(source, target string, conditions *storage.Conditions) error { +func (fs *GCSFs) copyFileInternal(source, target string, conditions *storage.Conditions, srcInfo os.FileInfo) error { src := fs.svc.Bucket(fs.config.Bucket).Object(source) dst := fs.svc.Bucket(fs.config.Bucket).Object(target) if conditions != nil { @@ -780,16 +784,20 @@ func (fs *GCSFs) copyFileInternal(source, target string, conditions *storage.Con if contentType != "" { copier.ContentType = contentType } + metadata := getMetadata(srcInfo) + if len(metadata) > 0 { + copier.Metadata = metadata + } _, err := copier.Run(ctx) metric.GCSCopyObjectCompleted(err) return err } -func (fs *GCSFs) renameInternal(source, target string, fi os.FileInfo, recursion int) (int, int64, error) { +func (fs *GCSFs) renameInternal(source, target string, srcInfo os.FileInfo, recursion int) (int, int64, error) { var numFiles int var filesSize int64 - if fi.IsDir() { + if srcInfo.IsDir() { if renameMode == 0 { hasContents, err := fs.hasContents(source) if err != nil { @@ -811,13 +819,13 @@ func (fs *GCSFs) renameInternal(source, target string, fi os.FileInfo, recursion } } } else { - if err := fs.copyFileInternal(source, target, nil); err != nil { + if err := fs.copyFileInternal(source, target, nil, srcInfo); err != nil { return numFiles, filesSize, err } numFiles++ - filesSize += fi.Size() + filesSize += srcInfo.Size() } - err := fs.Remove(source, fi.IsDir()) + err := fs.Remove(source, srcInfo.IsDir()) if fs.IsNotExist(err) { err = nil } @@ -1002,7 +1010,9 @@ func (l *gcsDirLister) Next(limit int) ([]os.FileInfo, error) { if val := getLastModified(attrs.Metadata); val > 0 { modTime = util.GetTimeFromMsecSinceEpoch(val) } - l.cache = append(l.cache, NewFileInfo(name, isDir, attrs.Size, modTime, false)) + info := NewFileInfo(name, isDir, attrs.Size, modTime, false) + info.setMetadata(attrs.Metadata) + l.cache = append(l.cache, info) } } diff --git a/internal/vfs/s3fs.go b/internal/vfs/s3fs.go index e50b6f1e..e6d9a609 100644 --- a/internal/vfs/s3fs.go +++ b/internal/vfs/s3fs.go @@ -167,7 +167,11 @@ func (fs *S3Fs) Stat(name string) (os.FileInfo, error) { _, err = fs.headObject(name + "/") isDir = err == nil } - return NewFileInfo(name, isDir, util.GetIntFromPointer(obj.ContentLength), util.GetTimeFromPointer(obj.LastModified), false), nil + info := NewFileInfo(name, isDir, util.GetIntFromPointer(obj.ContentLength), util.GetTimeFromPointer(obj.LastModified), false) + if !isDir { + info.setMetadata(obj.Metadata) + } + return info, nil } if !fs.IsNotExist(err) { return result, err @@ -632,9 +636,9 @@ func (fs *S3Fs) ResolvePath(virtualPath string) (string, error) { } // CopyFile implements the FsFileCopier interface -func (fs *S3Fs) CopyFile(source, target string, srcSize int64) (int, int64, error) { +func (fs *S3Fs) CopyFile(source, target string, srcInfo os.FileInfo) (int, int64, error) { numFiles := 1 - sizeDiff := srcSize + sizeDiff := srcInfo.Size() attrs, err := fs.headObject(target) if err == nil { sizeDiff -= util.GetIntFromPointer(attrs.ContentLength) @@ -644,7 +648,7 @@ func (fs *S3Fs) CopyFile(source, target string, srcSize int64) (int, int64, erro return 0, 0, err } } - if err := fs.copyFileInternal(source, target, srcSize); err != nil { + if err := fs.copyFileInternal(source, target, srcInfo); err != nil { return 0, 0, err } return numFiles, sizeDiff, nil @@ -682,14 +686,14 @@ func (fs *S3Fs) setConfigDefaults() { } } -func (fs *S3Fs) copyFileInternal(source, target string, fileSize int64) error { +func (fs *S3Fs) copyFileInternal(source, target string, srcInfo os.FileInfo) error { contentType := mime.TypeByExtension(path.Ext(source)) copySource := pathEscape(fs.Join(fs.config.Bucket, source)) - if fileSize > s3CopyObjectThreshold { + if srcInfo.Size() > s3CopyObjectThreshold { fsLog(fs, logger.LevelDebug, "renaming file %q with size %d using multipart copy", - source, fileSize) - err := fs.doMultipartCopy(copySource, target, contentType, fileSize) + source, srcInfo.Size()) + err := fs.doMultipartCopy(copySource, target, contentType, srcInfo.Size()) metric.S3CopyObjectCompleted(err) return err } @@ -703,17 +707,18 @@ func (fs *S3Fs) copyFileInternal(source, target string, fileSize int64) error { StorageClass: types.StorageClass(fs.config.StorageClass), ACL: types.ObjectCannedACL(fs.config.ACL), ContentType: util.NilIfEmpty(contentType), + Metadata: getMetadata(srcInfo), }) metric.S3CopyObjectCompleted(err) return err } -func (fs *S3Fs) renameInternal(source, target string, fi os.FileInfo, recursion int) (int, int64, error) { +func (fs *S3Fs) renameInternal(source, target string, srcInfo os.FileInfo, recursion int) (int, int64, error) { var numFiles int var filesSize int64 - if fi.IsDir() { + if srcInfo.IsDir() { if renameMode == 0 { hasContents, err := fs.hasContents(source) if err != nil { @@ -735,13 +740,13 @@ func (fs *S3Fs) renameInternal(source, target string, fi os.FileInfo, recursion } } } else { - if err := fs.copyFileInternal(source, target, fi.Size()); err != nil { + if err := fs.copyFileInternal(source, target, srcInfo); err != nil { return numFiles, filesSize, err } numFiles++ - filesSize += fi.Size() + filesSize += srcInfo.Size() } - err := fs.Remove(source, fi.IsDir()) + err := fs.Remove(source, srcInfo.IsDir()) if fs.IsNotExist(err) { err = nil } diff --git a/internal/vfs/vfs.go b/internal/vfs/vfs.go index a3f1290b..64ad4186 100644 --- a/internal/vfs/vfs.go +++ b/internal/vfs/vfs.go @@ -160,7 +160,7 @@ type FsRealPather interface { // FsFileCopier is a Fs that implements the CopyFile method. type FsFileCopier interface { Fs - CopyFile(source, target string, srcSize int64) (int, int64, error) + CopyFile(source, target string, srcInfo os.FileInfo) (int, int64, error) } // File defines an interface representing a SFTPGo file