beenull/sftpgo-mirror
1
0
Fork 0
mirror of https://github.com/drakkan/sftpgo.git synced 2024-11-25 00:50:31 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

preserve metadata on copy/rename

Browse source 
Signed-off-by: Nicola Murino <nicola.murino@gmail.com>
This commit is contained in:
Nicola Murino 2024-08-08 18:44:26 +02:00
parent b94451f731
commit a5c5e85144
No known key found for this signature in database
GPG key ID: 935D2952DEC4EECF
6 changed files with 108 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
internal/common/connection.go
View file

@ -620,7 +620,7 @@ func (c *BaseConnection) checkCopy(srcInfo, dstInfo os.FileInfo, virtualSource,
return nil
}
func (c *BaseConnection) copyFile(virtualSourcePath, virtualTargetPath string, srcSize int64) error {
func (c *BaseConnection) copyFile(virtualSourcePath, virtualTargetPath string, srcInfo os.FileInfo) error {
if !c.User.HasPerm(dataprovider.PermCopy, virtualSourcePath) || !c.User.HasPerm(dataprovider.PermCopy, virtualTargetPath) {
return c.GetPermissionDeniedError()
}
@ -638,12 +638,12 @@ func (c *BaseConnection) copyFile(virtualSourcePath, virtualTargetPath string, s
return err
}
startTime := time.Now()
numFiles, sizeDiff, err := copier.CopyFile(fsSourcePath, fsTargetPath, srcSize)
numFiles, sizeDiff, err := copier.CopyFile(fsSourcePath, fsTargetPath, srcInfo)
elapsed := time.Since(startTime).Nanoseconds() / 1000000
updateUserQuotaAfterFileWrite(c, virtualTargetPath, numFiles, sizeDiff)
logger.CommandLog(copyLogSender, fsSourcePath, fsTargetPath, c.User.Username, "", c.ID, c.protocol, -1, -1,
"", "", "", srcSize, c.localAddr, c.remoteAddr, elapsed)
ExecuteActionNotification(c, operationCopy, fsSourcePath, virtualSourcePath, fsTargetPath, virtualTargetPath, "", srcSize, err, elapsed, nil) //nolint:errcheck
"", "", "", srcInfo.Size(), c.localAddr, c.remoteAddr, elapsed)
ExecuteActionNotification(c, operationCopy, fsSourcePath, virtualSourcePath, fsTargetPath, virtualTargetPath, "", srcInfo.Size(), err, elapsed, nil) //nolint:errcheck
return err
}
}
@ -655,7 +655,7 @@ func (c *BaseConnection) copyFile(virtualSourcePath, virtualTargetPath string, s
defer rCancelFn()
defer reader.Close()
writer, numFiles, truncatedSize, wCancelFn, err := getFileWriter(c, virtualTargetPath, srcSize)
writer, numFiles, truncatedSize, wCancelFn, err := getFileWriter(c, virtualTargetPath, srcInfo.Size())
if err != nil {
return fmt.Errorf("unable to get writer for path %q: %w", virtualTargetPath, err)
}
@ -706,7 +706,7 @@ func (c *BaseConnection) doRecursiveCopy(virtualSourcePath, virtualTargetPath st
return nil
}
return c.copyFile(virtualSourcePath, virtualTargetPath, srcInfo.Size())
return c.copyFile(virtualSourcePath, virtualTargetPath, srcInfo)
}
func (c *BaseConnection) recursiveCopyEntries(virtualSourcePath, virtualTargetPath string, entries []os.FileInfo, recursion int) error {

44
internal/vfs/azblobfs.go
View file

@ -186,7 +186,11 @@ func (fs *AzureBlobFs) Stat(name string) (os.FileInfo, error) {
if val := getAzureLastModified(attrs.Metadata); val > 0 {
lastModified = util.GetTimeFromMsecSinceEpoch(val)
}
return NewFileInfo(name, isDir, util.GetIntFromPointer(attrs.ContentLength), lastModified, false), nil
info := NewFileInfo(name, isDir, util.GetIntFromPointer(attrs.ContentLength), lastModified, false)
if !isDir {
info.setMetadataFromPointerVal(attrs.Metadata)
}
return info, nil
}
if !fs.IsNotExist(err) {
return nil, err
@ -651,9 +655,9 @@ func (fs *AzureBlobFs) ResolvePath(virtualPath string) (string, error) {
}
// CopyFile implements the FsFileCopier interface
func (fs *AzureBlobFs) CopyFile(source, target string, srcSize int64) (int, int64, error) {
func (fs *AzureBlobFs) CopyFile(source, target string, srcInfo os.FileInfo) (int, int64, error) {
numFiles := 1
sizeDiff := srcSize
sizeDiff := srcInfo.Size()
attrs, err := fs.headObject(target)
if err == nil {
sizeDiff -= util.GetIntFromPointer(attrs.ContentLength)
@ -663,7 +667,7 @@ func (fs *AzureBlobFs) CopyFile(source, target string, srcSize int64) (int, int6
return 0, 0, err
}
}
if err := fs.copyFileInternal(source, target); err != nil {
if err := fs.copyFileInternal(source, target, srcInfo); err != nil {
return 0, 0, err
}
return numFiles, sizeDiff, nil
@ -746,13 +750,13 @@ func (fs *AzureBlobFs) setConfigDefaults() {
}
}
func (fs *AzureBlobFs) copyFileInternal(source, target string) error {
func (fs *AzureBlobFs) copyFileInternal(source, target string, srcInfo os.FileInfo) error {
ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxLongTimeout))
defer cancelFn()
srcBlob := fs.containerClient.NewBlockBlobClient(source)
dstBlob := fs.containerClient.NewBlockBlobClient(target)
resp, err := dstBlob.StartCopyFromURL(ctx, srcBlob.URL(), fs.getCopyOptions())
resp, err := dstBlob.StartCopyFromURL(ctx, srcBlob.URL(), fs.getCopyOptions(srcInfo))
if err != nil {
metric.AZCopyObjectCompleted(err)
return err
@ -785,11 +789,11 @@ func (fs *AzureBlobFs) copyFileInternal(source, target string) error {
return nil
}
func (fs *AzureBlobFs) renameInternal(source, target string, fi os.FileInfo, recursion int) (int, int64, error) {
func (fs *AzureBlobFs) renameInternal(source, target string, srcInfo os.FileInfo, recursion int) (int, int64, error) {
var numFiles int
var filesSize int64
if fi.IsDir() {
if srcInfo.IsDir() {
if renameMode == 0 {
hasContents, err := fs.hasContents(source)
if err != nil {
@ -811,13 +815,13 @@ func (fs *AzureBlobFs) renameInternal(source, target string, fi os.FileInfo, rec
}
}
} else {
if err := fs.copyFileInternal(source, target); err != nil {
if err := fs.copyFileInternal(source, target, srcInfo); err != nil {
return numFiles, filesSize, err
}
numFiles++
filesSize += fi.Size()
filesSize += srcInfo.Size()
}
err := fs.skipNotExistErr(fs.Remove(source, fi.IsDir()))
err := fs.skipNotExistErr(fs.Remove(source, srcInfo.IsDir()))
return numFiles, filesSize, err
}
@ -1098,11 +1102,20 @@ func (*AzureBlobFs) readFill(r io.Reader, buf []byte) (n int, err error) {
return n, err
}
func (fs *AzureBlobFs) getCopyOptions() *blob.StartCopyFromURLOptions {
func (fs *AzureBlobFs) getCopyOptions(srcInfo os.FileInfo) *blob.StartCopyFromURLOptions {
copyOptions := &blob.StartCopyFromURLOptions{}
if fs.config.AccessTier != "" {
copyOptions.Tier = (*blob.AccessTier)(&fs.config.AccessTier)
}
metadata := make(map[string]*string)
for k, v := range getMetadata(srcInfo) {
if v != "" {
metadata[k] = to.Ptr(v)
}
}
if len(metadata) > 0 {
copyOptions.Metadata = metadata
}
return copyOptions
}
@ -1254,6 +1267,7 @@ func (l *azureBlobDirLister) Next(limit int) ([]os.FileInfo, error) {
name = strings.TrimPrefix(name, l.prefix)
size := int64(0)
isDir := false
var metadata map[string]*string
modTime := time.Unix(0, 0)
if blobItem.Properties != nil {
size = util.GetIntFromPointer(blobItem.Properties.ContentLength)
@ -1266,12 +1280,16 @@ func (l *azureBlobDirLister) Next(limit int) ([]os.FileInfo, error) {
continue
}
l.prefixes[name] = true
} else {
metadata = blobItem.Metadata
}
if val := getAzureLastModified(blobItem.Metadata); val > 0 {
modTime = util.GetTimeFromMsecSinceEpoch(val)
}
}
l.cache = append(l.cache, NewFileInfo(name, isDir, size, modTime, false))
info := NewFileInfo(name, isDir, size, modTime, false)
info.setMetadataFromPointerVal(metadata)
l.cache = append(l.cache, info)
}
return l.returnFromCache(limit), nil

31
internal/vfs/fileinfo.go
View file

@ -18,6 +18,8 @@ import (
"os"
"path"
"time"
"github.com/drakkan/sftpgo/v2/internal/util"
)
// FileInfo implements os.FileInfo for a Cloud Storage file.
@ -26,6 +28,7 @@ type FileInfo struct {
sizeInBytes int64
modTime time.Time
mode os.FileMode
metadata map[string]string
}
// NewFileInfo creates file info.
@ -79,5 +82,33 @@ func (fi *FileInfo) SetMode(mode os.FileMode) {
// Sys provides the underlying data source (can return nil)
func (fi *FileInfo) Sys() any {
return fi.metadata
}
func (fi *FileInfo) setMetadata(value map[string]string) {
fi.metadata = value
}
func (fi *FileInfo) setMetadataFromPointerVal(value map[string]*string) {
if len(value) == 0 {
fi.metadata = nil
return
}
fi.metadata = map[string]string{}
for k, v := range value {
val := util.GetStringFromPointer(v)
if val != "" {
fi.metadata[k] = val
}
}
}
func getMetadata(fi os.FileInfo) map[string]string {
if val, ok := fi.Sys().(map[string]string); ok {
if len(val) > 0 {
return val
}
}
return nil
}

32
internal/vfs/gcsfs.go
View file

@ -636,9 +636,9 @@ func (fs *GCSFs) ResolvePath(virtualPath string) (string, error) {
}
// CopyFile implements the FsFileCopier interface
func (fs *GCSFs) CopyFile(source, target string, srcSize int64) (int, int64, error) {
func (fs *GCSFs) CopyFile(source, target string, srcInfo os.FileInfo) (int, int64, error) {
numFiles := 1
sizeDiff := srcSize
sizeDiff := srcInfo.Size()
var conditions *storage.Conditions
attrs, err := fs.headObject(target)
if err == nil {
@ -651,7 +651,7 @@ func (fs *GCSFs) CopyFile(source, target string, srcSize int64) (int, int64, err
}
conditions = &storage.Conditions{DoesNotExist: true}
}
if err := fs.copyFileInternal(source, target, conditions); err != nil {
if err := fs.copyFileInternal(source, target, conditions, srcInfo); err != nil {
return 0, 0, err
}
return numFiles, sizeDiff, nil
@ -679,7 +679,11 @@ func (fs *GCSFs) getObjectStat(name string) (os.FileInfo, error) {
objectModTime = util.GetTimeFromMsecSinceEpoch(val)
}
isDir := attrs.ContentType == dirMimeType || strings.HasSuffix(attrs.Name, "/")
return NewFileInfo(name, isDir, objSize, objectModTime, false), nil
info := NewFileInfo(name, isDir, objSize, objectModTime, false)
if !isDir {
info.setMetadata(attrs.Metadata)
}
return info, nil
}
if !fs.IsNotExist(err) {
return nil, err
@ -749,7 +753,7 @@ func (fs *GCSFs) composeObjects(ctx context.Context, dst, partialObject *storage
return err
}
func (fs *GCSFs) copyFileInternal(source, target string, conditions *storage.Conditions) error {
func (fs *GCSFs) copyFileInternal(source, target string, conditions *storage.Conditions, srcInfo os.FileInfo) error {
src := fs.svc.Bucket(fs.config.Bucket).Object(source)
dst := fs.svc.Bucket(fs.config.Bucket).Object(target)
if conditions != nil {
@ -780,16 +784,20 @@ func (fs *GCSFs) copyFileInternal(source, target string, conditions *storage.Con
if contentType != "" {
copier.ContentType = contentType
}
metadata := getMetadata(srcInfo)
if len(metadata) > 0 {
copier.Metadata = metadata
}
_, err := copier.Run(ctx)
metric.GCSCopyObjectCompleted(err)
return err
}
func (fs *GCSFs) renameInternal(source, target string, fi os.FileInfo, recursion int) (int, int64, error) {
func (fs *GCSFs) renameInternal(source, target string, srcInfo os.FileInfo, recursion int) (int, int64, error) {
var numFiles int
var filesSize int64
if fi.IsDir() {
if srcInfo.IsDir() {
if renameMode == 0 {
hasContents, err := fs.hasContents(source)
if err != nil {
@ -811,13 +819,13 @@ func (fs *GCSFs) renameInternal(source, target string, fi os.FileInfo, recursion
}
}
} else {
if err := fs.copyFileInternal(source, target, nil); err != nil {
if err := fs.copyFileInternal(source, target, nil, srcInfo); err != nil {
return numFiles, filesSize, err
}
numFiles++
filesSize += fi.Size()
filesSize += srcInfo.Size()
}
err := fs.Remove(source, fi.IsDir())
err := fs.Remove(source, srcInfo.IsDir())
if fs.IsNotExist(err) {
err = nil
}
@ -1002,7 +1010,9 @@ func (l *gcsDirLister) Next(limit int) ([]os.FileInfo, error) {
if val := getLastModified(attrs.Metadata); val > 0 {
modTime = util.GetTimeFromMsecSinceEpoch(val)
}
l.cache = append(l.cache, NewFileInfo(name, isDir, attrs.Size, modTime, false))
info := NewFileInfo(name, isDir, attrs.Size, modTime, false)
info.setMetadata(attrs.Metadata)
l.cache = append(l.cache, info)
}
}

31
internal/vfs/s3fs.go
View file

@ -167,7 +167,11 @@ func (fs *S3Fs) Stat(name string) (os.FileInfo, error) {
_, err = fs.headObject(name + "/")
isDir = err == nil
}
return NewFileInfo(name, isDir, util.GetIntFromPointer(obj.ContentLength), util.GetTimeFromPointer(obj.LastModified), false), nil
info := NewFileInfo(name, isDir, util.GetIntFromPointer(obj.ContentLength), util.GetTimeFromPointer(obj.LastModified), false)
if !isDir {
info.setMetadata(obj.Metadata)
}
return info, nil
}
if !fs.IsNotExist(err) {
return result, err
@ -632,9 +636,9 @@ func (fs *S3Fs) ResolvePath(virtualPath string) (string, error) {
}
// CopyFile implements the FsFileCopier interface
func (fs *S3Fs) CopyFile(source, target string, srcSize int64) (int, int64, error) {
func (fs *S3Fs) CopyFile(source, target string, srcInfo os.FileInfo) (int, int64, error) {
numFiles := 1
sizeDiff := srcSize
sizeDiff := srcInfo.Size()
attrs, err := fs.headObject(target)
if err == nil {
sizeDiff -= util.GetIntFromPointer(attrs.ContentLength)
@ -644,7 +648,7 @@ func (fs *S3Fs) CopyFile(source, target string, srcSize int64) (int, int64, erro
return 0, 0, err
}
}
if err := fs.copyFileInternal(source, target, srcSize); err != nil {
if err := fs.copyFileInternal(source, target, srcInfo); err != nil {
return 0, 0, err
}
return numFiles, sizeDiff, nil
@ -682,14 +686,14 @@ func (fs *S3Fs) setConfigDefaults() {
}
}
func (fs *S3Fs) copyFileInternal(source, target string, fileSize int64) error {
func (fs *S3Fs) copyFileInternal(source, target string, srcInfo os.FileInfo) error {
contentType := mime.TypeByExtension(path.Ext(source))
copySource := pathEscape(fs.Join(fs.config.Bucket, source))
if fileSize > s3CopyObjectThreshold {
if srcInfo.Size() > s3CopyObjectThreshold {
fsLog(fs, logger.LevelDebug, "renaming file %q with size %d using multipart copy",
source, fileSize)
err := fs.doMultipartCopy(copySource, target, contentType, fileSize)
source, srcInfo.Size())
err := fs.doMultipartCopy(copySource, target, contentType, srcInfo.Size())
metric.S3CopyObjectCompleted(err)
return err
}
@ -703,17 +707,18 @@ func (fs *S3Fs) copyFileInternal(source, target string, fileSize int64) error {
StorageClass: types.StorageClass(fs.config.StorageClass),
ACL: types.ObjectCannedACL(fs.config.ACL),
ContentType: util.NilIfEmpty(contentType),
Metadata: getMetadata(srcInfo),
})
metric.S3CopyObjectCompleted(err)
return err
}
func (fs *S3Fs) renameInternal(source, target string, fi os.FileInfo, recursion int) (int, int64, error) {
func (fs *S3Fs) renameInternal(source, target string, srcInfo os.FileInfo, recursion int) (int, int64, error) {
var numFiles int
var filesSize int64
if fi.IsDir() {
if srcInfo.IsDir() {
if renameMode == 0 {
hasContents, err := fs.hasContents(source)
if err != nil {
@ -735,13 +740,13 @@ func (fs *S3Fs) renameInternal(source, target string, fi os.FileInfo, recursion
}
}
} else {
if err := fs.copyFileInternal(source, target, fi.Size()); err != nil {
if err := fs.copyFileInternal(source, target, srcInfo); err != nil {
return numFiles, filesSize, err
}
numFiles++
filesSize += fi.Size()
filesSize += srcInfo.Size()
}
err := fs.Remove(source, fi.IsDir())
err := fs.Remove(source, srcInfo.IsDir())
if fs.IsNotExist(err) {
err = nil
}

2
internal/vfs/vfs.go
View file

@ -160,7 +160,7 @@ type FsRealPather interface {
// FsFileCopier is a Fs that implements the CopyFile method.
type FsFileCopier interface {
Fs
CopyFile(source, target string, srcSize int64) (int, int64, error)
CopyFile(source, target string, srcInfo os.FileInfo) (int, int64, error)
}
// File defines an interface representing a SFTPGo file