beenull/sftpgo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

eventmanager: check disk quota before executing the compress action

Browse source 
Signed-off-by: Nicola Murino <nicola.murino@gmail.com>
This commit is contained in:
Nicola Murino 2022-12-31 16:41:32 +01:00
parent 15ad31da54
commit 6cebc037a0
No known key found for this signature in database
GPG key ID: 935D2952DEC4EECF
5 changed files with 259 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
go.mod
View file

@ -17,7 +17,7 @@ require (
github.com/aws/aws-sdk-go-v2/service/s3 v1.29.6
github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.17.0
github.com/aws/aws-sdk-go-v2/service/sts v1.17.7
github.com/cockroachdb/cockroach-go/v2 v2.2.19
github.com/cockroachdb/cockroach-go/v2 v2.2.20
github.com/coreos/go-oidc/v3 v3.4.0
github.com/drakkan/webdav v0.0.0-20221101181759-17ed21f9337b
github.com/eikenb/pipeat v0.0.0-20210730190139-06b3e6902001

5
go.sum
View file

@ -360,8 +360,8 @@ github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWH
github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
github.com/cockroachdb/cockroach-go/v2 v2.2.19 h1:YIHyz17jZumBeXPuoZKq/0nrITsqDoDD8/KQt3/xiyc=
github.com/cockroachdb/cockroach-go/v2 v2.2.19/go.mod h1:mzlIDDBALQfEjv/7DU12fb2AfQ/MUYTlychcMpWp9QI=
github.com/cockroachdb/cockroach-go/v2 v2.2.20 h1:TLSzwdTdIwgsbdApHzaxunhSMrmbGf5YY6oxtaP2kvw=
github.com/cockroachdb/cockroach-go/v2 v2.2.20/go.mod h1:73vQi5H/H7kE8SgOt+XA6729Tubvj5hxKIEgbQQhp4c=
github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo=
github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA=
@ -1337,7 +1337,6 @@ github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqgg
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=

53
internal/common/eventmanager.go
View file

@ -1615,6 +1615,50 @@ func getArchiveBaseDir(paths []string) string {
return baseDir
}
func getSizeForPath(conn *BaseConnection, p string, info os.FileInfo) (int64, error) {
if info.IsDir() {
var dirSize int64
entries, err := conn.ListDir(p)
if err != nil {
return 0, err
}
for _, entry := range entries {
size, err := getSizeForPath(conn, path.Join(p, entry.Name()), entry)
if err != nil {
return 0, err
}
dirSize += size
}
return dirSize, nil
}
if info.Mode().IsRegular() {
return info.Size(), nil
}
return 0, nil
}
func estimateZipSize(conn *BaseConnection, zipPath string, paths []string) (int64, error) {
q, _ := conn.HasSpace(false, false, zipPath)
if q.HasSpace && q.GetRemainingSize() > 0 {
var size int64
for _, item := range paths {
info, err := conn.DoStat(item, 1, false)
if err != nil {
return size, err
}
itemSize, err := getSizeForPath(conn, item, info)
if err != nil {
return size, err
}
size += itemSize
}
eventManagerLog(logger.LevelDebug, "archive paths %v, archive name %q, size: %d", paths, zipPath, size)
// we assume the zip size will be half of the real size
return size / 2, nil
}
return -1, nil
}
func executeCompressFsActionForUser(c dataprovider.EventActionFsCompress, replacer *strings.Replacer,
user dataprovider.User,
) error {
@ -1638,14 +1682,19 @@ func executeCompressFsActionForUser(c dataprovider.EventActionFsCompress, replac
}
paths = append(paths, p)
}
writer, numFiles, truncatedSize, cancelFn, err := getFileWriter(conn, name, -1)
paths = util.RemoveDuplicates(paths, false)
estimatedSize, err := estimateZipSize(conn, name, paths)
if err != nil {
eventManagerLog(logger.LevelError, "unable to estimate size for archive %q: %v", name, err)
return fmt.Errorf("unable to estimate archive size: %w", err)
}
writer, numFiles, truncatedSize, cancelFn, err := getFileWriter(conn, name, estimatedSize)
if err != nil {
eventManagerLog(logger.LevelError, "unable to create archive %q: %v", name, err)
return fmt.Errorf("unable to create archive: %w", err)
}
defer cancelFn()
paths = util.RemoveDuplicates(paths, false)
baseDir := getArchiveBaseDir(paths)
eventManagerLog(logger.LevelDebug, "creating archive %q for paths %+v", name, paths)

37
internal/common/eventmanager_test.go
View file

@ -1720,6 +1720,43 @@ func TestReplacePathsPlaceholders(t *testing.T) {
assert.Equal(t, []string{"/path1", "/path2"}, paths)
}
func TestEstimateZipSizeErrors(t *testing.T) {
u := dataprovider.User{
BaseUser: sdk.BaseUser{
Username: "u",
HomeDir: filepath.Join(os.TempDir(), "u"),
Status: 1,
Permissions: map[string][]string{
"/": {dataprovider.PermAny},
},
QuotaSize: 1000,
},
}
err := dataprovider.AddUser(&u, "", "", "")
assert.NoError(t, err)
err = os.MkdirAll(u.GetHomeDir(), os.ModePerm)
assert.NoError(t, err)
conn := NewBaseConnection("", ProtocolFTP, "", "", u)
_, err = getSizeForPath(conn, "/missing", vfs.NewFileInfo("missing", true, 0, time.Now(), false))
assert.True(t, conn.IsNotExistError(err))
if runtime.GOOS != osWindows {
err = os.MkdirAll(filepath.Join(u.HomeDir, "d1", "d2", "sub"), os.ModePerm)
assert.NoError(t, err)
err = os.WriteFile(filepath.Join(u.HomeDir, "d1", "d2", "sub", "file.txt"), []byte("data"), 0666)
assert.NoError(t, err)
err = os.Chmod(filepath.Join(u.HomeDir, "d1", "d2"), 0001)
assert.NoError(t, err)
size, err := estimateZipSize(conn, "/archive.zip", []string{"/d1"})
assert.Error(t, err, "size %d", size)
err = os.Chmod(filepath.Join(u.HomeDir, "d1", "d2"), os.ModePerm)
assert.NoError(t, err)
}
err = dataprovider.DeleteUser(u.Username, "", "", "")
assert.NoError(t, err)
err = os.RemoveAll(u.GetHomeDir())
assert.NoError(t, err)
}
func getErrorString(err error) string {
if err == nil {
return ""

168
internal/common/protocol_test.go
View file

@ -4714,6 +4714,174 @@ func TestEventActionCompress(t *testing.T) {
assert.NoError(t, err)
}
func TestEventActionCompressQuotaErrors(t *testing.T) {
smtpCfg := smtp.Config{
Host: "127.0.0.1",
Port: 2525,
From: "notify@example.com",
TemplatesPath: "templates",
}
err := smtpCfg.Initialize(configDir)
require.NoError(t, err)
testDir := "archiveDir"
zipPath := "/archive.zip"
a1 := dataprovider.BaseEventAction{
Name: "action1",
Type: dataprovider.ActionTypeFilesystem,
Options: dataprovider.BaseEventActionOptions{
FsConfig: dataprovider.EventActionFilesystemConfig{
Type: dataprovider.FilesystemActionCompress,
Compress: dataprovider.EventActionFsCompress{
Name: zipPath,
Paths: []string{"/" + testDir},
},
},
},
}
action1, _, err := httpdtest.AddEventAction(a1, http.StatusCreated)
assert.NoError(t, err)
a2 := dataprovider.BaseEventAction{
Name: "action2",
Type: dataprovider.ActionTypeEmail,
Options: dataprovider.BaseEventActionOptions{
EmailConfig: dataprovider.EventActionEmailConfig{
Recipients: []string{"test@example.com"},
Subject: `"Compress failed"`,
Body: "Error: {{ErrorString}}",
},
},
}
action2, _, err := httpdtest.AddEventAction(a2, http.StatusCreated)
assert.NoError(t, err)
r1 := dataprovider.EventRule{
Name: "test compress",
Trigger: dataprovider.EventTriggerFsEvent,
Conditions: dataprovider.EventConditions{
FsEvents: []string{"rename"},
},
Actions: []dataprovider.EventAction{
{
BaseEventAction: dataprovider.BaseEventAction{
Name: action1.Name,
},
Order: 1,
},
{
BaseEventAction: dataprovider.BaseEventAction{
Name: action2.Name,
},
Options: dataprovider.EventActionOptions{
IsFailureAction: true,
},
Order: 2,
},
},
}
rule1, _, err := httpdtest.AddEventRule(r1, http.StatusCreated)
assert.NoError(t, err)
fileSize := int64(100)
u := getTestUser()
u.QuotaSize = 10 * fileSize
user, _, err := httpdtest.AddUser(u, http.StatusCreated)
assert.NoError(t, err)
conn, client, err := getSftpClient(user)
if assert.NoError(t, err) {
defer conn.Close()
defer client.Close()
err = client.MkdirAll(path.Join(testDir, "1", "1"))
assert.NoError(t, err)
err = writeSFTPFile(path.Join(testDir, "1", testFileName), fileSize, client)
assert.NoError(t, err)
err = client.MkdirAll(path.Join(testDir, "2", "2"))
assert.NoError(t, err)
err = writeSFTPFile(path.Join(testDir, "2", testFileName), fileSize, client)
assert.NoError(t, err)
err = client.Symlink(path.Join(testDir, "2", testFileName), path.Join(testDir, "2", testFileName+"_link"))
assert.NoError(t, err)
// trigger the compress action
err = client.Mkdir("a")
assert.NoError(t, err)
err = client.Rename("a", "b")
assert.NoError(t, err)
assert.Eventually(t, func() bool {
_, err := client.Stat(zipPath)
return err == nil
}, 3*time.Second, 100*time.Millisecond)
err = client.Remove(zipPath)
assert.NoError(t, err)
// add other 6 file, the compress action should fail with a quota error
err = writeSFTPFile(path.Join(testDir, "1", "1", testFileName), fileSize, client)
assert.NoError(t, err)
err = writeSFTPFile(path.Join(testDir, "2", "2", testFileName), fileSize, client)
assert.NoError(t, err)
err = writeSFTPFile(path.Join(testDir, "1", "1", testFileName+"1"), fileSize, client)
assert.NoError(t, err)
err = writeSFTPFile(path.Join(testDir, "2", "2", testFileName+"2"), fileSize, client)
assert.NoError(t, err)
err = writeSFTPFile(path.Join(testDir, "1", testFileName+"1"), fileSize, client)
assert.NoError(t, err)
err = writeSFTPFile(path.Join(testDir, "2", testFileName+"2"), fileSize, client)
assert.NoError(t, err)
lastReceivedEmail.reset()
err = client.Rename("b", "a")
assert.NoError(t, err)
assert.Eventually(t, func() bool {
return lastReceivedEmail.get().From != ""
}, 3*time.Second, 100*time.Millisecond)
email := lastReceivedEmail.get()
assert.Len(t, email.To, 1)
assert.True(t, util.Contains(email.To, "test@example.com"))
assert.Contains(t, email.Data, `Subject: "Compress failed"`)
assert.Contains(t, email.Data, common.ErrQuotaExceeded.Error())
// update quota size so the user is already overquota
user.QuotaSize = 7 * fileSize
_, _, err = httpdtest.UpdateUser(user, http.StatusOK, "")
assert.NoError(t, err)
lastReceivedEmail.reset()
err = client.Rename("a", "b")
assert.NoError(t, err)
assert.Eventually(t, func() bool {
return lastReceivedEmail.get().From != ""
}, 3*time.Second, 100*time.Millisecond)
email = lastReceivedEmail.get()
assert.Len(t, email.To, 1)
assert.True(t, util.Contains(email.To, "test@example.com"))
assert.Contains(t, email.Data, `Subject: "Compress failed"`)
assert.Contains(t, email.Data, common.ErrQuotaExceeded.Error())
// remove the path to compress to trigger an error for size estimation
out, err := runSSHCommand(fmt.Sprintf("sftpgo-remove %s", testDir), user)
assert.NoError(t, err, string(out))
lastReceivedEmail.reset()
err = client.Rename("b", "a")
assert.NoError(t, err)
assert.Eventually(t, func() bool {
return lastReceivedEmail.get().From != ""
}, 3*time.Second, 100*time.Millisecond)
email = lastReceivedEmail.get()
assert.Len(t, email.To, 1)
assert.True(t, util.Contains(email.To, "test@example.com"))
assert.Contains(t, email.Data, `Subject: "Compress failed"`)
assert.Contains(t, email.Data, "unable to estimate archive size")
}
_, err = httpdtest.RemoveEventRule(rule1, http.StatusOK)
assert.NoError(t, err)
_, err = httpdtest.RemoveEventAction(action1, http.StatusOK)
assert.NoError(t, err)
_, err = httpdtest.RemoveEventAction(action2, http.StatusOK)
assert.NoError(t, err)
_, err = httpdtest.RemoveUser(user, http.StatusOK)
assert.NoError(t, err)
err = os.RemoveAll(user.GetHomeDir())
assert.NoError(t, err)
smtpCfg = smtp.Config{}
err = smtpCfg.Initialize(configDir)
require.NoError(t, err)
}
func TestEventActionCompressQuotaFolder(t *testing.T) {
testDir := "/folder"
a1 := dataprovider.BaseEventAction{