From 04dc97072bd74f6871efdbb942bda1c611c608d9 Mon Sep 17 00:00:00 2001 From: Nicola Murino Date: Mon, 26 Sep 2022 19:00:34 +0200 Subject: [PATCH] eventmanager: add metadata check Signed-off-by: Nicola Murino --- docs/eventmanager.md | 1 + go.mod | 6 +-- go.sum | 11 ++-- internal/common/common.go | 74 ++++++++++++++++++++++++-- internal/common/common_test.go | 15 ++++++ internal/common/eventmanager.go | 63 ++++++++++++++++++++++- internal/common/eventmanager_test.go | 50 ++++++++++++++++++ internal/common/protocol_test.go | 77 ++++++++++++++++++++++++++++ internal/dataprovider/eventrule.go | 9 ++-- internal/httpd/api_metadata.go | 71 ++----------------------- internal/httpd/internal_test.go | 8 +-- 11 files changed, 297 insertions(+), 88 deletions(-) diff --git a/docs/eventmanager.md b/docs/eventmanager.md index 8e7b7e74..afbd361c 100644 --- a/docs/eventmanager.md +++ b/docs/eventmanager.md @@ -12,6 +12,7 @@ The following actions are supported: - `Folder quota reset`. The quota used by virtual folders will be updated based on current usage. - `Transfer quota reset`. The transfer quota values will be reset to `0`. - `Data retention check`. You can define per-folder retention policies. +- `Metadata check`. A metadata check requires a metadata plugin such as [this one](https://github.com/sftpgo/sftpgo-plugin-metadata) and removes the metadata associated to missing items (for example objects deleted outside SFTPGo). A metadata check does nothing is no metadata plugin is installed or external metadata are not supported for a filesystem. - `Filesystem`. For these actions, the required permissions are automatically granted. This is the same as executing the actions from an SFTP client and the same restrictions applies. Supported actions: - `Rename`. You can rename one or more files or directories. - `Delete`. You can delete one or more files and directories. diff --git a/go.mod b/go.mod index b6a5f799..846fd8c9 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,7 @@ require ( github.com/hashicorp/go-plugin v1.4.5 github.com/hashicorp/go-retryablehttp v0.7.1 github.com/jlaffaye/ftp v0.0.0-20201112195030-9aae4d151126 - github.com/klauspost/compress v1.15.10 + github.com/klauspost/compress v1.15.11 github.com/lestrrat-go/jwx v1.2.25 github.com/lib/pq v1.10.7 github.com/lithammer/shortuuid/v3 v3.0.7 @@ -68,7 +68,7 @@ require ( golang.org/x/crypto v0.0.0-20220924013350-4ba4fb4dd9e7 golang.org/x/net v0.0.0-20220923203811-8be639271d50 golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1 - golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 + golang.org/x/sys v0.0.0-20220926163933-8cfa568d3c25 golang.org/x/time v0.0.0-20220922220347-f3bd1da661af google.golang.org/api v0.97.0 gopkg.in/natefinch/lumberjack.v2 v2.0.0 @@ -127,7 +127,7 @@ require ( github.com/magiconair/properties v1.8.6 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.16 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect github.com/miekg/dns v1.1.50 // indirect github.com/minio/sha256-simd v1.0.0 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect diff --git a/go.sum b/go.sum index 6aa70696..845ba4e0 100644 --- a/go.sum +++ b/go.sum @@ -548,8 +548,8 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.15.10 h1:Ai8UzuomSCDw90e1qNMtb15msBXsNpH6gzkkENQNcJo= -github.com/klauspost/compress v1.15.10/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= +github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c= +github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.1.1 h1:t0wUqjowdm8ezddV5k0tLWVklVuvLJpoHeb4WBdydm0= github.com/klauspost/cpuid/v2 v2.1.1/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= @@ -617,8 +617,9 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= -github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09JkG9WeqChjprR5s9bBZ+OM= +github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mhale/smtpd v0.8.0 h1:5JvdsehCg33PQrZBvFyDMMUDQmvbzVpZgKob7eYBJc0= github.com/mhale/smtpd v0.8.0/go.mod h1:MQl+y2hwIEQCXtNhe5+55n0GZOjSmeqORDIXbqUL3x4= github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= @@ -980,8 +981,8 @@ golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 h1:h+EGohizhe9XlX18rfpa8k8RAc5XyaeamM+0VHRd4lc= -golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220926163933-8cfa568d3c25 h1:nwzwVf0l2Y/lkov/+IYgMMbFyI+QypZDds9RxlSmsFQ= +golang.org/x/sys v0.0.0-20220926163933-8cfa568d3c25/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/internal/common/common.go b/internal/common/common.go index 0f248649..17347b2a 100644 --- a/internal/common/common.go +++ b/internal/common/common.go @@ -143,9 +143,11 @@ var ( // Connections is the list of active connections Connections ActiveConnections // QuotaScans is the list of active quota scans - QuotaScans ActiveScans - transfersChecker TransfersChecker - supportedProtocols = []string{ProtocolSFTP, ProtocolSCP, ProtocolSSH, ProtocolFTP, ProtocolWebDAV, + QuotaScans ActiveScans + // ActiveMetadataChecks holds the active metadata checks + ActiveMetadataChecks MetadataChecks + transfersChecker TransfersChecker + supportedProtocols = []string{ProtocolSFTP, ProtocolSCP, ProtocolSSH, ProtocolFTP, ProtocolWebDAV, ProtocolHTTP, ProtocolHTTPShare, ProtocolOIDC} disconnHookProtocols = []string{ProtocolSFTP, ProtocolSCP, ProtocolSSH, ProtocolFTP} // the map key is the protocol, for each protocol we can have multiple rate limiters @@ -1158,7 +1160,7 @@ func (c *ConnectionStatus) GetTransfersAsString() string { return result } -// ActiveQuotaScan defines an active quota scan for a user home dir +// ActiveQuotaScan defines an active quota scan for a user type ActiveQuotaScan struct { // Username to which the quota scan refers Username string `json:"username"` @@ -1181,7 +1183,7 @@ type ActiveScans struct { FolderScans []ActiveVirtualFolderQuotaScan } -// GetUsersQuotaScans returns the active quota scans for users home directories +// GetUsersQuotaScans returns the active users quota scans func (s *ActiveScans) GetUsersQuotaScans() []ActiveQuotaScan { s.RLock() defer s.RUnlock() @@ -1271,3 +1273,65 @@ func (s *ActiveScans) RemoveVFolderQuotaScan(folderName string) bool { return false } + +// MetadataCheck defines an active metadata check +type MetadataCheck struct { + // Username to which the metadata check refers + Username string `json:"username"` + // check start time as unix timestamp in milliseconds + StartTime int64 `json:"start_time"` +} + +// MetadataChecks holds the active metadata checks +type MetadataChecks struct { + sync.RWMutex + checks []MetadataCheck +} + +// Get returns the active metadata checks +func (c *MetadataChecks) Get() []MetadataCheck { + c.RLock() + defer c.RUnlock() + + checks := make([]MetadataCheck, len(c.checks)) + copy(checks, c.checks) + + return checks +} + +// Add adds a user to the ones with active metadata checks. +// Return false if a metadata check is already active for the specified user +func (c *MetadataChecks) Add(username string) bool { + c.Lock() + defer c.Unlock() + + for idx := range c.checks { + if c.checks[idx].Username == username { + return false + } + } + + c.checks = append(c.checks, MetadataCheck{ + Username: username, + StartTime: util.GetTimeAsMsSinceEpoch(time.Now()), + }) + + return true +} + +// Remove removes a user from the ones with active metadata checks +func (c *MetadataChecks) Remove(username string) bool { + c.Lock() + defer c.Unlock() + + for idx := range c.checks { + if c.checks[idx].Username == username { + lastIdx := len(c.checks) - 1 + c.checks[idx] = c.checks[lastIdx] + c.checks = c.checks[:lastIdx] + return true + } + } + + return false +} diff --git a/internal/common/common_test.go b/internal/common/common_test.go index 839b41df..1c4a4343 100644 --- a/internal/common/common_test.go +++ b/internal/common/common_test.go @@ -1228,6 +1228,21 @@ func TestUpdateTransferTimestamps(t *testing.T) { assert.NoError(t, err) } +func TestMetadataAPI(t *testing.T) { + username := "metadatauser" + require.False(t, ActiveMetadataChecks.Remove(username)) + require.True(t, ActiveMetadataChecks.Add(username)) + require.False(t, ActiveMetadataChecks.Add(username)) + checks := ActiveMetadataChecks.Get() + require.Len(t, checks, 1) + checks[0].Username = username + "a" + checks = ActiveMetadataChecks.Get() + require.Len(t, checks, 1) + require.Equal(t, username, checks[0].Username) + require.True(t, ActiveMetadataChecks.Remove(username)) + require.Len(t, ActiveMetadataChecks.Get(), 0) +} + func BenchmarkBcryptHashing(b *testing.B) { bcryptPassword := "bcryptpassword" for i := 0; i < b.N; i++ { diff --git a/internal/common/eventmanager.go b/internal/common/eventmanager.go index 00e42250..029936c2 100644 --- a/internal/common/eventmanager.go +++ b/internal/common/eventmanager.go @@ -1561,7 +1561,66 @@ func executeDataRetentionCheckRuleAction(config dataprovider.EventActionDataRete return nil } -func executeRuleAction(action dataprovider.BaseEventAction, params *EventParams, conditions dataprovider.ConditionOptions) error { +func executeMetadataCheckForUser(user dataprovider.User) error { + if err := user.LoadAndApplyGroupSettings(); err != nil { + eventManagerLog(logger.LevelDebug, "skipping scheduled quota reset for user %s, cannot apply group settings: %v", + user.Username, err) + return err + } + if !ActiveMetadataChecks.Add(user.Username) { + eventManagerLog(logger.LevelError, "another metadata check is already in progress for user %q", user.Username) + return fmt.Errorf("another metadata check is in progress for user %q", user.Username) + } + defer ActiveMetadataChecks.Remove(user.Username) + + if err := user.CheckMetadataConsistency(); err != nil { + eventManagerLog(logger.LevelError, "error checking metadata consistence for user %q: %v", user.Username, err) + return fmt.Errorf("error checking metadata consistence for user %q: %w", user.Username, err) + } + return nil +} + +func executeMetadataCheckRuleAction(conditions dataprovider.ConditionOptions, params *EventParams) error { + users, err := params.getUsers() + if err != nil { + return fmt.Errorf("unable to get users: %w", err) + } + var failures []string + var executed int + for _, user := range users { + // if sender is set, the conditions have already been evaluated + if params.sender == "" { + if !checkEventConditionPatterns(user.Username, conditions.Names) { + eventManagerLog(logger.LevelDebug, "skipping metadata check for user %q, name conditions don't match", + user.Username) + continue + } + if !checkEventGroupConditionPatters(user.Groups, conditions.GroupNames) { + eventManagerLog(logger.LevelDebug, "skipping metadata check for user %q, group name conditions don't match", + user.Username) + continue + } + } + executed++ + if err = executeMetadataCheckForUser(user); err != nil { + params.AddError(err) + failures = append(failures, user.Username) + continue + } + } + if len(failures) > 0 { + return fmt.Errorf("metadata check failed for users: %+v", failures) + } + if executed == 0 { + eventManagerLog(logger.LevelError, "no metadata check executed") + return errors.New("no metadata check executed") + } + return nil +} + +func executeRuleAction(action dataprovider.BaseEventAction, params *EventParams, + conditions dataprovider.ConditionOptions, +) error { var err error switch action.Type { @@ -1581,6 +1640,8 @@ func executeRuleAction(action dataprovider.BaseEventAction, params *EventParams, err = executeTransferQuotaResetRuleAction(conditions, params) case dataprovider.ActionTypeDataRetentionCheck: err = executeDataRetentionCheckRuleAction(action.Options.RetentionConfig, conditions, params, action.Name) + case dataprovider.ActionTypeMetadataCheck: + err = executeMetadataCheckRuleAction(conditions, params) case dataprovider.ActionTypeFilesystem: err = executeFsRuleAction(action.Options.FsConfig, conditions, params) default: diff --git a/internal/common/eventmanager_test.go b/internal/common/eventmanager_test.go index 6f34a764..a7224530 100644 --- a/internal/common/eventmanager_test.go +++ b/internal/common/eventmanager_test.go @@ -323,6 +323,8 @@ func TestEventManagerErrors(t *testing.T) { assert.Error(t, err) err = executeTransferQuotaResetRuleAction(dataprovider.ConditionOptions{}, &EventParams{}) assert.Error(t, err) + err = executeMetadataCheckRuleAction(dataprovider.ConditionOptions{}, &EventParams{}) + assert.Error(t, err) err = executeDeleteFsRuleAction(nil, nil, dataprovider.ConditionOptions{}, &EventParams{}) assert.Error(t, err) err = executeMkdirFsRuleAction(nil, nil, dataprovider.ConditionOptions{}, &EventParams{}) @@ -342,6 +344,15 @@ func TestEventManagerErrors(t *testing.T) { }, }) assert.Error(t, err) + err = executeMetadataCheckForUser(dataprovider.User{ + Groups: []sdk.GroupMapping{ + { + Name: groupName, + Type: sdk.GroupTypePrimary, + }, + }, + }) + assert.Error(t, err) err = executeDataRetentionCheckForUser(dataprovider.User{ Groups: []sdk.GroupMapping{ { @@ -649,6 +660,41 @@ func TestEventRuleActions(t *testing.T) { assert.Contains(t, err.Error(), "no user quota reset executed") } + action = dataprovider.BaseEventAction{ + Type: dataprovider.ActionTypeMetadataCheck, + } + + err = executeRuleAction(action, &EventParams{}, dataprovider.ConditionOptions{ + Names: []dataprovider.ConditionPattern{ + { + Pattern: "don't match", + }, + }, + }) + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "no metadata check executed") + } + + err = executeRuleAction(action, &EventParams{}, dataprovider.ConditionOptions{ + Names: []dataprovider.ConditionPattern{ + { + Pattern: username1, + }, + }, + }) + assert.NoError(t, err) + // simulate another metadata check in progress + assert.True(t, ActiveMetadataChecks.Add(username1)) + err = executeRuleAction(action, &EventParams{}, dataprovider.ConditionOptions{ + Names: []dataprovider.ConditionPattern{ + { + Pattern: username1, + }, + }, + }) + assert.Error(t, err) + assert.True(t, ActiveMetadataChecks.Remove(username1)) + dataRetentionAction := dataprovider.BaseEventAction{ Type: dataprovider.ActionTypeDataRetentionCheck, Options: dataprovider.BaseEventActionOptions{ @@ -988,6 +1034,10 @@ func TestEventRuleActionsNoGroupMatching(t *testing.T) { if assert.Error(t, err) { assert.Contains(t, err.Error(), "no user quota reset executed") } + err = executeMetadataCheckRuleAction(conditions, &EventParams{}) + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "no metadata check executed") + } err = executeTransferQuotaResetRuleAction(conditions, &EventParams{}) if assert.Error(t, err) { assert.Contains(t, err.Error(), "no transfer quota reset executed") diff --git a/internal/common/protocol_test.go b/internal/common/protocol_test.go index 56b624c4..1236446e 100644 --- a/internal/common/protocol_test.go +++ b/internal/common/protocol_test.go @@ -5345,6 +5345,15 @@ func TestSplittedRenamePerms(t *testing.T) { } func TestSFTPLoopError(t *testing.T) { + smtpCfg := smtp.Config{ + Host: "127.0.0.1", + Port: 2525, + From: "notification@example.com", + TemplatesPath: "templates", + } + err := smtpCfg.Initialize(configDir) + require.NoError(t, err) + user1 := getTestUser() user2 := getTestUser() user1.Username += "1" @@ -5381,6 +5390,63 @@ func TestSFTPLoopError(t *testing.T) { assert.NoError(t, err, string(resp)) user2, resp, err = httpdtest.AddUser(user2, http.StatusCreated) assert.NoError(t, err, string(resp)) + // test metadata check event error + a1 := dataprovider.BaseEventAction{ + Name: "a1", + Type: dataprovider.ActionTypeMetadataCheck, + } + action1, _, err := httpdtest.AddEventAction(a1, http.StatusCreated) + assert.NoError(t, err) + a2 := dataprovider.BaseEventAction{ + Name: "a2", + Type: dataprovider.ActionTypeEmail, + Options: dataprovider.BaseEventActionOptions{ + EmailConfig: dataprovider.EventActionEmailConfig{ + Recipients: []string{"failure@example.com"}, + Subject: `Failed action"`, + Body: "Test body", + }, + }, + } + action2, _, err := httpdtest.AddEventAction(a2, http.StatusCreated) + assert.NoError(t, err) + r1 := dataprovider.EventRule{ + Name: "rule1", + Trigger: dataprovider.EventTriggerProviderEvent, + Conditions: dataprovider.EventConditions{ + ProviderEvents: []string{"update"}, + }, + Actions: []dataprovider.EventAction{ + { + BaseEventAction: dataprovider.BaseEventAction{ + Name: action1.Name, + }, + Order: 1, + }, + { + BaseEventAction: dataprovider.BaseEventAction{ + Name: action2.Name, + }, + Order: 2, + Options: dataprovider.EventActionOptions{ + IsFailureAction: true, + }, + }, + }, + } + rule1, _, err := httpdtest.AddEventRule(r1, http.StatusCreated) + assert.NoError(t, err) + + lastReceivedEmail.reset() + _, _, err = httpdtest.UpdateUser(user2, http.StatusOK, "") + assert.NoError(t, err) + assert.Eventually(t, func() bool { + return lastReceivedEmail.get().From != "" + }, 3000*time.Millisecond, 100*time.Millisecond) + email := lastReceivedEmail.get() + assert.Len(t, email.To, 1) + assert.True(t, util.Contains(email.To, "failure@example.com")) + assert.Contains(t, email.Data, `Subject: Failed action`) user1.VirtualFolders[0].FsConfig.SFTPConfig.Password = kms.NewPlainSecret(defaultPassword) user2.FsConfig.SFTPConfig.Password = kms.NewPlainSecret(defaultPassword) @@ -5399,6 +5465,13 @@ func TestSFTPLoopError(t *testing.T) { if assert.Error(t, err) { assert.Contains(t, err.Error(), "SFTP loop") } + + _, err = httpdtest.RemoveEventRule(rule1, http.StatusOK) + assert.NoError(t, err) + _, err = httpdtest.RemoveEventAction(action1, http.StatusOK) + assert.NoError(t, err) + _, err = httpdtest.RemoveEventAction(action2, http.StatusOK) + assert.NoError(t, err) _, err = httpdtest.RemoveUser(user1, http.StatusOK) assert.NoError(t, err) err = os.RemoveAll(user1.GetHomeDir()) @@ -5409,6 +5482,10 @@ func TestSFTPLoopError(t *testing.T) { assert.NoError(t, err) _, err = httpdtest.RemoveFolder(vfs.BaseVirtualFolder{Name: "sftp"}, http.StatusOK) assert.NoError(t, err) + + smtpCfg = smtp.Config{} + err = smtpCfg.Initialize(configDir) + require.NoError(t, err) } func TestNonLocalCrossRename(t *testing.T) { diff --git a/internal/dataprovider/eventrule.go b/internal/dataprovider/eventrule.go index 5afcc293..24b29b02 100644 --- a/internal/dataprovider/eventrule.go +++ b/internal/dataprovider/eventrule.go @@ -44,12 +44,13 @@ const ( ActionTypeTransferQuotaReset ActionTypeDataRetentionCheck ActionTypeFilesystem + ActionTypeMetadataCheck ) var ( supportedEventActions = []int{ActionTypeHTTP, ActionTypeCommand, ActionTypeEmail, ActionTypeBackup, ActionTypeUserQuotaReset, ActionTypeFolderQuotaReset, ActionTypeTransferQuotaReset, - ActionTypeDataRetentionCheck, ActionTypeFilesystem} + ActionTypeDataRetentionCheck, ActionTypeMetadataCheck, ActionTypeFilesystem} ) func isActionTypeValid(action int) bool { @@ -72,6 +73,8 @@ func getActionTypeAsString(action int) string { return "Transfer quota reset" case ActionTypeDataRetentionCheck: return "Data retention check" + case ActionTypeMetadataCheck: + return "Metadata check" case ActionTypeFilesystem: return "Filesystem" default: @@ -1257,7 +1260,7 @@ func (r *EventRule) validate() error { func (r *EventRule) checkIPBlockedAndCertificateActions() error { unavailableActions := []int{ActionTypeUserQuotaReset, ActionTypeFolderQuotaReset, ActionTypeTransferQuotaReset, - ActionTypeDataRetentionCheck, ActionTypeFilesystem} + ActionTypeDataRetentionCheck, ActionTypeMetadataCheck, ActionTypeFilesystem} for _, action := range r.Actions { if util.Contains(unavailableActions, action.Type) { return fmt.Errorf("action %q, type %q is not supported for event trigger %q", @@ -1272,7 +1275,7 @@ func (r *EventRule) checkProviderEventActions(providerObjectType string) error { // can be executed only if we modify a user. They will be executed for the // affected user. Folder quota reset can be executed only for folders. userSpecificActions := []int{ActionTypeUserQuotaReset, ActionTypeTransferQuotaReset, - ActionTypeDataRetentionCheck, ActionTypeFilesystem} + ActionTypeDataRetentionCheck, ActionTypeMetadataCheck, ActionTypeFilesystem} for _, action := range r.Actions { if util.Contains(userSpecificActions, action.Type) && providerObjectType != actionObjectUser { return fmt.Errorf("action %q, type %q is only supported for provider user events", diff --git a/internal/httpd/api_metadata.go b/internal/httpd/api_metadata.go index 7bce3c9d..b7127606 100644 --- a/internal/httpd/api_metadata.go +++ b/internal/httpd/api_metadata.go @@ -17,80 +17,17 @@ package httpd import ( "fmt" "net/http" - "sync" - "time" "github.com/go-chi/render" + "github.com/drakkan/sftpgo/v2/internal/common" "github.com/drakkan/sftpgo/v2/internal/dataprovider" "github.com/drakkan/sftpgo/v2/internal/logger" - "github.com/drakkan/sftpgo/v2/internal/util" ) -var ( - activeMetadataChecks metadataChecks -) - -type metadataCheck struct { - // Username to which the metadata check refers - Username string `json:"username"` - // check start time as unix timestamp in milliseconds - StartTime int64 `json:"start_time"` -} - -// metadataChecks holds the active metadata checks -type metadataChecks struct { - sync.RWMutex - checks []metadataCheck -} - -func (c *metadataChecks) get() []metadataCheck { - c.RLock() - defer c.RUnlock() - - checks := make([]metadataCheck, len(c.checks)) - copy(checks, c.checks) - - return checks -} - -func (c *metadataChecks) add(username string) bool { - c.Lock() - defer c.Unlock() - - for idx := range c.checks { - if c.checks[idx].Username == username { - return false - } - } - - c.checks = append(c.checks, metadataCheck{ - Username: username, - StartTime: util.GetTimeAsMsSinceEpoch(time.Now()), - }) - - return true -} - -func (c *metadataChecks) remove(username string) bool { - c.Lock() - defer c.Unlock() - - for idx := range c.checks { - if c.checks[idx].Username == username { - lastIdx := len(c.checks) - 1 - c.checks[idx] = c.checks[lastIdx] - c.checks = c.checks[:lastIdx] - return true - } - } - - return false -} - func getMetadataChecks(w http.ResponseWriter, r *http.Request) { r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize) - render.JSON(w, r, activeMetadataChecks.get()) + render.JSON(w, r, common.ActiveMetadataChecks.Get()) } func startMetadataCheck(w http.ResponseWriter, r *http.Request) { @@ -101,7 +38,7 @@ func startMetadataCheck(w http.ResponseWriter, r *http.Request) { sendAPIResponse(w, r, err, "", getRespStatus(err)) return } - if !activeMetadataChecks.add(user.Username) { + if !common.ActiveMetadataChecks.Add(user.Username) { sendAPIResponse(w, r, err, fmt.Sprintf("Another check is already in progress for user %#v", user.Username), http.StatusConflict) return @@ -112,7 +49,7 @@ func startMetadataCheck(w http.ResponseWriter, r *http.Request) { } func doMetadataCheck(user dataprovider.User) error { - defer activeMetadataChecks.remove(user.Username) + defer common.ActiveMetadataChecks.Remove(user.Username) err := user.CheckMetadataConsistency() if err != nil { diff --git a/internal/httpd/internal_test.go b/internal/httpd/internal_test.go index 5d3abe4d..9a53cc09 100644 --- a/internal/httpd/internal_test.go +++ b/internal/httpd/internal_test.go @@ -2402,7 +2402,7 @@ func TestUserCanResetPassword(t *testing.T) { func TestMetadataAPI(t *testing.T) { username := "metadatauser" - assert.False(t, activeMetadataChecks.remove(username)) + assert.False(t, common.ActiveMetadataChecks.Remove(username)) user := dataprovider.User{ BaseUser: sdk.BaseUser{ @@ -2417,7 +2417,7 @@ func TestMetadataAPI(t *testing.T) { err := dataprovider.AddUser(&user, "", "") assert.NoError(t, err) - assert.True(t, activeMetadataChecks.add(username)) + assert.True(t, common.ActiveMetadataChecks.Add(username)) req, err := http.NewRequest(http.MethodPost, path.Join(metadataBasePath, username, "check"), nil) assert.NoError(t, err) @@ -2429,8 +2429,8 @@ func TestMetadataAPI(t *testing.T) { startMetadataCheck(rr, req) assert.Equal(t, http.StatusConflict, rr.Code) - assert.True(t, activeMetadataChecks.remove(username)) - assert.Len(t, activeMetadataChecks.get(), 0) + assert.True(t, common.ActiveMetadataChecks.Remove(username)) + assert.Len(t, common.ActiveMetadataChecks.Get(), 0) err = dataprovider.DeleteUser(username, "", "") assert.NoError(t, err)