From a587228cf0c8039591b895d453a70e5b89256128 Mon Sep 17 00:00:00 2001 From: Nicola Murino Date: Thu, 16 Dec 2021 18:18:36 +0100 Subject: [PATCH] add support for metadata plugins --- common/common.go | 5 +- common/connection.go | 17 +- common/connection_test.go | 15 +- common/transfer.go | 2 +- dataprovider/admin.go | 4 +- dataprovider/user.go | 22 + docs/full-configuration.md | 6 +- docs/plugins.md | 7 +- ftpd/handler.go | 2 +- go.mod | 20 +- go.sum | 94 ++- httpd/api_http_user.go | 2 +- httpd/api_metadata.go | 110 +++ httpd/httpd.go | 2 + httpd/httpd_test.go | 47 ++ httpd/internal_test.go | 39 + httpd/server.go | 3 + openapi/openapi.yaml | 74 ++ pkgs/build.sh | 2 +- sdk/plugin/eventsearcher/eventsearcher.go | 2 +- sdk/plugin/eventsearcher/grpc.go | 2 +- sdk/plugin/metadata.go | 82 ++ sdk/plugin/metadata/grpc.go | 160 ++++ sdk/plugin/metadata/metadata.go | 61 ++ sdk/plugin/metadata/proto/metadata.pb.go | 938 ++++++++++++++++++++++ sdk/plugin/metadata/proto/metadata.proto | 54 ++ sdk/plugin/mkproto.sh | 1 + sdk/plugin/plugin.go | 158 +++- sftpd/handler.go | 2 +- vfs/azblobfs.go | 132 ++- vfs/folder.go | 12 + vfs/gcsfs.go | 119 ++- vfs/osfs.go | 7 +- vfs/s3fs.go | 95 ++- vfs/sftpfs.go | 7 +- vfs/vfs.go | 108 ++- webdavd/handler.go | 2 +- 37 files changed, 2283 insertions(+), 132 deletions(-) create mode 100644 httpd/api_metadata.go create mode 100644 sdk/plugin/metadata.go create mode 100644 sdk/plugin/metadata/grpc.go create mode 100644 sdk/plugin/metadata/metadata.go create mode 100644 sdk/plugin/metadata/proto/metadata.pb.go create mode 100644 sdk/plugin/metadata/proto/metadata.proto diff --git a/common/common.go b/common/common.go index 00d43718..d603ff8e 100644 --- a/common/common.go +++ b/common/common.go @@ -369,8 +369,9 @@ type Configuration struct { Actions ProtocolActions `json:"actions" mapstructure:"actions"` // SetstatMode 0 means "normal mode": requests for changing permissions and owner/group are executed. // 1 means "ignore mode": requests for changing permissions and owner/group are silently ignored. - // 2 means "ignore mode for cloud fs": requests for changing permissions and owner/group/time are - // silently ignored for cloud based filesystem such as S3, GCS, Azure Blob + // 2 means "ignore mode for cloud fs": requests for changing permissions and owner/group are + // silently ignored for cloud based filesystem such as S3, GCS, Azure Blob. Requests for changing + // modification times are ignored for cloud based filesystem if they are not supported. SetstatMode int `json:"setstat_mode" mapstructure:"setstat_mode"` // TempPath defines the path for temporary files such as those used for atomic uploads or file pipes. // If you set this option you must make sure that the defined path exists, is accessible for writing diff --git a/common/connection.go b/common/connection.go index 1a8a4472..566725ba 100644 --- a/common/connection.go +++ b/common/connection.go @@ -202,15 +202,16 @@ func (c *BaseConnection) getRealFsPath(fsPath string) string { return fsPath } -func (c *BaseConnection) setTimes(fsPath string, atime time.Time, mtime time.Time) { +func (c *BaseConnection) setTimes(fsPath string, atime time.Time, mtime time.Time) bool { c.RLock() defer c.RUnlock() for _, t := range c.activeTransfers { if t.SetTimes(fsPath, atime, mtime) { - return + return true } } + return false } func (c *BaseConnection) truncateOpenHandle(fsPath string, size int64) (int64, error) { @@ -293,7 +294,7 @@ func (c *BaseConnection) RemoveFile(fs vfs.Fs, fsPath, virtualPath string, info c.Log(logger.LevelDebug, "remove for path %#v handled by pre-delete action", fsPath) } else { if err := fs.Remove(fsPath, false); err != nil { - c.Log(logger.LevelWarn, "failed to remove a file/symlink %#v: %+v", fsPath, err) + c.Log(logger.LevelWarn, "failed to remove file/symlink %#v: %+v", fsPath, err) return c.GetFsError(fs, err) } } @@ -562,15 +563,19 @@ func (c *BaseConnection) handleChtimes(fs vfs.Fs, fsPath, pathForPerms string, a if !c.User.HasPerm(dataprovider.PermChtimes, pathForPerms) { return c.GetPermissionDeniedError() } - if c.ignoreSetStat(fs) { + if Config.SetstatMode == 1 { return nil } - if err := fs.Chtimes(c.getRealFsPath(fsPath), attributes.Atime, attributes.Mtime); err != nil { + isUploading := c.setTimes(fsPath, attributes.Atime, attributes.Mtime) + if err := fs.Chtimes(c.getRealFsPath(fsPath), attributes.Atime, attributes.Mtime, isUploading); err != nil { + c.setTimes(fsPath, time.Time{}, time.Time{}) + if errors.Is(err, vfs.ErrVfsUnsupported) && Config.SetstatMode == 2 { + return nil + } c.Log(logger.LevelWarn, "failed to chtimes for path %#v, access time: %v, modification time: %v, err: %+v", fsPath, attributes.Atime, attributes.Mtime, err) return c.GetFsError(fs, err) } - c.setTimes(fsPath, attributes.Atime, attributes.Mtime) accessTimeString := attributes.Atime.Format(chtimesFormat) modificationTimeString := attributes.Mtime.Format(chtimesFormat) logger.CommandLog(chtimesLogSender, fsPath, "", c.User.Username, "", c.ID, c.protocol, -1, -1, diff --git a/common/connection_test.go b/common/connection_test.go index f96e93d2..1ab0e5f0 100644 --- a/common/connection_test.go +++ b/common/connection_test.go @@ -23,19 +23,23 @@ type MockOsFs struct { } // Name returns the name for the Fs implementation -func (fs MockOsFs) Name() string { +func (fs *MockOsFs) Name() string { return "mockOsFs" } // HasVirtualFolders returns true if folders are emulated -func (fs MockOsFs) HasVirtualFolders() bool { +func (fs *MockOsFs) HasVirtualFolders() bool { return fs.hasVirtualFolders } -func (fs MockOsFs) IsUploadResumeSupported() bool { +func (fs *MockOsFs) IsUploadResumeSupported() bool { return !fs.hasVirtualFolders } +func (fs *MockOsFs) Chtimes(name string, atime, mtime time.Time, isUploading bool) error { + return vfs.ErrVfsUnsupported +} + func newMockOsFs(hasVirtualFolders bool, connectionID, rootDir string) vfs.Fs { return &MockOsFs{ Fs: vfs.NewOsFs(connectionID, rootDir, ""), @@ -99,6 +103,11 @@ func TestSetStatMode(t *testing.T) { Config.SetstatMode = 2 err = conn.handleChmod(fs, fakePath, fakePath, nil) assert.NoError(t, err) + err = conn.handleChtimes(fs, fakePath, fakePath, &StatAttributes{ + Atime: time.Now(), + Mtime: time.Now(), + }) + assert.NoError(t, err) Config.SetstatMode = oldSetStatMode } diff --git a/common/transfer.go b/common/transfer.go index 0700deb3..89c80329 100644 --- a/common/transfer.go +++ b/common/transfer.go @@ -280,7 +280,7 @@ func (t *BaseTransfer) Close() error { func (t *BaseTransfer) updateTimes() { if !t.aTime.IsZero() && !t.mTime.IsZero() { - err := t.Fs.Chtimes(t.fsPath, t.aTime, t.mTime) + err := t.Fs.Chtimes(t.fsPath, t.aTime, t.mTime, true) t.Connection.Log(logger.LevelDebug, "set times for file %#v, atime: %v, mtime: %v, err: %v", t.fsPath, t.aTime, t.mTime, err) } diff --git a/dataprovider/admin.go b/dataprovider/admin.go index 5b8aca7a..a6684028 100644 --- a/dataprovider/admin.go +++ b/dataprovider/admin.go @@ -39,6 +39,7 @@ const ( PermAdminManageDefender = "manage_defender" PermAdminViewDefender = "view_defender" PermAdminRetentionChecks = "retention_checks" + PermAdminMetadataChecks = "metadata_checks" PermAdminViewEvents = "view_events" ) @@ -47,7 +48,8 @@ var ( validAdminPerms = []string{PermAdminAny, PermAdminAddUsers, PermAdminChangeUsers, PermAdminDeleteUsers, PermAdminViewUsers, PermAdminViewConnections, PermAdminCloseConnections, PermAdminViewServerStatus, PermAdminManageAdmins, PermAdminManageAPIKeys, PermAdminQuotaScans, PermAdminManageSystem, - PermAdminManageDefender, PermAdminViewDefender, PermAdminRetentionChecks, PermAdminViewEvents} + PermAdminManageDefender, PermAdminViewDefender, PermAdminRetentionChecks, PermAdminMetadataChecks, + PermAdminViewEvents} ) // TOTPConfig defines the time-based one time password configuration diff --git a/dataprovider/user.go b/dataprovider/user.go index ec91f8bc..d1cb977f 100644 --- a/dataprovider/user.go +++ b/dataprovider/user.go @@ -447,6 +447,27 @@ func (u *User) GetVirtualFolderForPath(virtualPath string) (vfs.VirtualFolder, e return folder, errNoMatchingVirtualFolder } +// CheckMetadataConsistency checks the consistency between the metadata stored +// in the configured metadata plugin and the filesystem +func (u *User) CheckMetadataConsistency() error { + fs, err := u.getRootFs("") + if err != nil { + return err + } + defer fs.Close() + + if err = fs.CheckMetadata(); err != nil { + return err + } + for idx := range u.VirtualFolders { + v := &u.VirtualFolders[idx] + if err = v.CheckMetadataConsistency(); err != nil { + return err + } + } + return nil +} + // ScanQuota scans the user home dir and virtual folders, included in its quota, // and returns the number of files and their size func (u *User) ScanQuota() (int, int64, error) { @@ -455,6 +476,7 @@ func (u *User) ScanQuota() (int, int64, error) { return 0, 0, err } defer fs.Close() + numFiles, size, err := fs.ScanRootDirContents() if err != nil { return numFiles, size, err diff --git a/docs/full-configuration.md b/docs/full-configuration.md index 9434eba7..d3deafca 100644 --- a/docs/full-configuration.md +++ b/docs/full-configuration.md @@ -58,7 +58,7 @@ The configuration file contains the following sections: - `execute_on`, list of strings. Valid values are `pre-download`, `download`, `pre-upload`, `upload`, `pre-delete`, `delete`, `rename`, `ssh_cmd`. Leave empty to disable actions. - `execute_sync`, list of strings. Actions to be performed synchronously. The `pre-delete` action is always executed synchronously while the other ones are asynchronous. Executing an action synchronously means that SFTPGo will not return a result code to the client (which is waiting for it) until your hook have completed its execution. Leave empty to execute only the `pre-delete` hook synchronously - `hook`, string. Absolute path to the command to execute or HTTP URL to notify. - - `setstat_mode`, integer. 0 means "normal mode": requests for changing permissions, owner/group and access/modification times are executed. 1 means "ignore mode": requests for changing permissions, owner/group and access/modification times are silently ignored. 2 means "ignore mode for cloud based filesystems": requests for changing permissions, owner/group and access/modification times are silently ignored for cloud filesystems and executed for local filesystem. + - `setstat_mode`, integer. 0 means "normal mode": requests for changing permissions, owner/group and access/modification times are executed. 1 means "ignore mode": requests for changing permissions, owner/group and access/modification times are silently ignored. 2 means "ignore mode if not supported": requests for changing permissions and owner/group are silently ignored for cloud filesystems and executed for local/SFTP filesystem. Requests for changing modification times are always executed for local/SFTP filesystems and are executed for cloud based filesystems if the target is a file and there is a metadata plugin available. A metadata plugin can be found [here](https://github.com/sftpgo/sftpgo-plugin-metadata). - `temp_path`, string. Defines the path for temporary files such as those used for atomic uploads or file pipes. If you set this option you must make sure that the defined path exists, is accessible for writing by the user running SFTPGo, and is on the same filesystem as the users home directories otherwise the renaming for atomic uploads will become a copy and therefore may take a long time. The temporary files are not namespaced. The default is generally fine. Leave empty for the default. - `proxy_protocol`, integer. Support for [HAProxy PROXY protocol](https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt). If you are running SFTPGo behind a proxy server such as HAProxy, AWS ELB or NGNIX, you can enable the proxy protocol. It provides a convenient way to safely transport connection information such as a client's address across multiple layers of NAT or TCP proxies to get the real client IP address instead of the proxy IP. Both protocol versions 1 and 2 are supported. If the proxy protocol is enabled in SFTPGo then you have to enable the protocol in your proxy configuration too. For example, for HAProxy, add `send-proxy` or `send-proxy-v2` to each server configuration line. The following modes are supported: - 0, disabled @@ -291,7 +291,7 @@ The configuration file contains the following sections: - `domain`, string. Domain to use for `HELO` command, if empty `localhost` will be used. Default: empty. - `templates_path`, string. Path to the email templates. This can be an absolute path or a path relative to the config dir. Templates are searched within a subdirectory named "email" in the specified path. You can customize the email templates by simply specifying an alternate path and putting your custom templates there. - **plugins**, list of external plugins. Each plugin is configured using a struct with the following fields: - - `type`, string. Defines the plugin type. Supported types: `notifier`, `kms`, `auth`. + - `type`, string. Defines the plugin type. Supported types: `notifier`, `kms`, `auth`, `metadata`. - `notifier_options`, struct. Defines the options for notifier plugins. - `fs_events`, list of strings. Defines the filesystem events that will be notified to this plugin. - `provider_events`, list of strings. Defines the provider events that will be notified to this plugin. @@ -308,7 +308,7 @@ The configuration file contains the following sections: - `sha256sum`, string. SHA256 checksum for the plugin executable. If not empty it will be used to verify the integrity of the executable. - `auto_mtls`, boolean. If enabled the client and the server automatically negotiate mutual TLS for transport authentication. This ensures that only the original client will be allowed to connect to the server, and all other connections will be rejected. The client will also refuse to connect to any server that isn't the original instance started by the client. -Please note that the plugin system is experimental, the exposed configuration parameters and interfaces may change in a backward incompatible way in future. +:warning: Please note that the plugin system is experimental, the exposed configuration parameters and interfaces may change in a backward incompatible way in future. A full example showing the default config (in JSON format) can be found [here](../sftpgo.json). diff --git a/docs/plugins.md b/docs/plugins.md index 085cd520..f36bdfec 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -10,11 +10,14 @@ For added security you can enable the automatic TLS. In this way, the client and The following plugin types are supported: -- `auth`, allows to authenticate users +- `auth`, allows to authenticate users. - `notifier`, allows to receive notifications for supported filesystem events such as file uploads, downloads etc. and provider events such as objects add, update, delete. - `kms`, allows to support additional KMS providers. +- `metadata`, allows to store metadata, such as the last modification time, for storage backends that does not support them (S3, Google Cloud Storage, Azure Blob). -Full configuration details can be found [here](./full-configuration.md) +Full configuration details can be found [here](./full-configuration.md). + +:warning: Please note that the plugin system is experimental, the exposed configuration parameters and interfaces may change in a backward incompatible way in future. ## Available plugins diff --git a/ftpd/handler.go b/ftpd/handler.go index 2ca7c6a6..3c6970c6 100644 --- a/ftpd/handler.go +++ b/ftpd/handler.go @@ -113,7 +113,7 @@ func (c *Connection) Remove(name string) error { var fi os.FileInfo if fi, err = fs.Lstat(p); err != nil { - c.Log(logger.LevelWarn, "failed to remove a file %#v: stat error: %+v", p, err) + c.Log(logger.LevelWarn, "failed to remove file %#v: stat error: %+v", p, err) return c.GetFsError(fs, err) } diff --git a/go.mod b/go.mod index e79efdbc..657afbe9 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/Azure/azure-storage-blob-go v0.14.0 github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962 github.com/alexedwards/argon2id v0.0.0-20211130144151-3585854a6387 - github.com/aws/aws-sdk-go v1.42.22 + github.com/aws/aws-sdk-go v1.42.23 github.com/cockroachdb/cockroach-go/v2 v2.2.5 github.com/eikenb/pipeat v0.0.0-20210603033007-44fc3ffce52b github.com/fclairamb/ftpserverlib v0.16.0 @@ -38,11 +38,11 @@ require ( github.com/prometheus/client_golang v1.11.0 github.com/rs/cors v1.8.0 github.com/rs/xid v1.3.0 - github.com/rs/zerolog v1.26.0 + github.com/rs/zerolog v1.26.1 github.com/shirou/gopsutil/v3 v3.21.11 github.com/spf13/afero v1.6.0 - github.com/spf13/cobra v1.2.1 - github.com/spf13/viper v1.9.0 + github.com/spf13/cobra v1.3.0 + github.com/spf13/viper v1.10.1 github.com/stretchr/testify v1.7.0 github.com/studio-b12/gowebdav v0.0.0-20211106090535-29e74efa701f github.com/wagslane/go-password-validator v0.3.0 @@ -51,12 +51,12 @@ require ( go.etcd.io/bbolt v1.3.6 go.uber.org/automaxprocs v1.4.0 gocloud.dev v0.24.0 - golang.org/x/crypto v0.0.0-20211202192323-5770296d904e + golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e golang.org/x/net v0.0.0-20211209124913-491a49abca63 - golang.org/x/sys v0.0.0-20211210111614-af8b64212486 + golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 - google.golang.org/api v0.62.0 - google.golang.org/grpc v1.42.0 + google.golang.org/api v0.63.0 + google.golang.org/grpc v1.43.0 google.golang.org/protobuf v1.27.1 gopkg.in/natefinch/lumberjack.v2 v2.0.0 ) @@ -70,7 +70,7 @@ require ( github.com/census-instrumentation/opencensus-proto v0.3.0 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4 // indirect - github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490 // indirect + github.com/cncf/xds/go v0.0.0-20211216145620-d92e9ce0af51 // indirect github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -139,6 +139,6 @@ replace ( github.com/eikenb/pipeat => github.com/drakkan/pipeat v0.0.0-20210805162858-70e57fa8a639 github.com/fclairamb/ftpserverlib => github.com/drakkan/ftpserverlib v0.0.0-20211107071448-34ff70e85dfb github.com/jlaffaye/ftp => github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9 - golang.org/x/crypto => github.com/drakkan/crypto v0.0.0-20211203175531-87c7ca02d2a9 + golang.org/x/crypto => github.com/drakkan/crypto v0.0.0-20211216170250-0a05a5747f0f golang.org/x/net => github.com/drakkan/net v0.0.0-20211210172952-3f0f9446f73f ) diff --git a/go.sum b/go.sum index 02c3195e..c973da02 100644 --- a/go.sum +++ b/go.sum @@ -44,9 +44,8 @@ cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4g cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/firestore v1.5.0/go.mod h1:c4nNYR1qdq7eaZ+jSc5fonrQN2k3M7sWATcYTiakjEo= -cloud.google.com/go/firestore v1.6.0/go.mod h1:afJwI0vaXwAG54kI7A//lP/lSPDkQORQuMkv56TxEPU= +cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= cloud.google.com/go/kms v0.1.0 h1:VXAb5OzejDcyhFzIDeZ5n5AUdlsFnCyexuascIwWMj0= cloud.google.com/go/kms v0.1.0/go.mod h1:8Qp8PCAypHg4FdmlyW1QRAv09BGQ9Uzh7JnmIZxPk+c= cloud.google.com/go/monitoring v0.1.0/go.mod h1:Hpm3XfzJv+UTiXzCG5Ffp0wijzHTC7Cv4eR7o3x/fEE= @@ -112,6 +111,7 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962 h1:KeNholpO2xKjgaaSyd+DyQRrsQjhbSeS7qe4nEw8aQw= github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962/go.mod h1:kC29dT1vFpj7py2OvG1khBdQpo3kInWP+6QipLbdngo= github.com/GoogleCloudPlatform/cloudsql-proxy v1.24.0/go.mod h1:3tx938GhY4FC+E1KT/jNjDw7Z5qxAEtIiERJ2sXjnII= @@ -131,14 +131,15 @@ github.com/alexedwards/argon2id v0.0.0-20211130144151-3585854a6387/go.mod h1:GuR github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.38.68/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.40.34/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= -github.com/aws/aws-sdk-go v1.42.22 h1:EwcM7/+Ytg6xK+jbeM2+f9OELHqPiEiEKetT/GgAr7I= -github.com/aws/aws-sdk-go v1.42.22/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= +github.com/aws/aws-sdk-go v1.42.23 h1:V0V5hqMEyVelgpu1e4gMPVCJ+KhmscdNxP/NWP1iCOA= +github.com/aws/aws-sdk-go v1.42.23/go.mod h1:gyRszuZ/icHmHAVE4gc/r+cfCmhA1AD+vqfWbgI+eHs= github.com/aws/aws-sdk-go-v2 v1.7.0/go.mod h1:tb9wi5s61kTDA5qCkcDbt3KRVV74GGslQkl/DRdX/P4= github.com/aws/aws-sdk-go-v2 v1.9.0/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= github.com/aws/aws-sdk-go-v2/config v1.7.0/go.mod h1:w9+nMZ7soXCe5nT46Ri354SNhXDQ6v+V5wqDjnZE+GY= @@ -160,7 +161,6 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/boombuler/barcode v1.0.1 h1:NDBbPmhS+EqABEs5Kg3n/5ZNjy73Pz7SIV+KCeqyXcs= github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= @@ -178,6 +178,8 @@ github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= +github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= @@ -190,8 +192,9 @@ github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490 h1:KwaoQzs/WeUxxJqiJsZ4euOly1Az/IgZXXSxlD/UBNk= github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211216145620-d92e9ce0af51 h1:F6fR7MjvOIk+FLQOeBCAbbKItVgbdj0l9VWPiHeBEiY= +github.com/cncf/xds/go v0.0.0-20211216145620-d92e9ce0af51/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/cockroach-go/v2 v2.2.5 h1:tfPdGHO5YpmrpN2ikJZYpaSGgU8WALwwjH3s+msiTQ0= github.com/cockroachdb/cockroach-go/v2 v2.2.5/go.mod h1:q4ZRgO6CQpwNyEvEwSxwNrOSVchsmzrBnAv3HuZ3Abc= @@ -202,7 +205,6 @@ github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+ github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= @@ -219,8 +221,8 @@ github.com/devigned/tab v0.1.1/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mz github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= -github.com/drakkan/crypto v0.0.0-20211203175531-87c7ca02d2a9 h1:vZ3cl6F+IEw7NI7yMrC3UdOl92R6nK+OGJz41RdOLPc= -github.com/drakkan/crypto v0.0.0-20211203175531-87c7ca02d2a9/go.mod h1:SiM6ypd8Xu1xldObYtbDztuUU7xUzMnUULfphXFZmro= +github.com/drakkan/crypto v0.0.0-20211216170250-0a05a5747f0f h1:12WWFMrTzDfKo/7sDtkQyuRhIanAavJdbrq9hYmlEgM= +github.com/drakkan/crypto v0.0.0-20211216170250-0a05a5747f0f/go.mod h1:SiM6ypd8Xu1xldObYtbDztuUU7xUzMnUULfphXFZmro= github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9 h1:LPH1dEblAOO/LoG7yHPMtBLXhQmjaga91/DDjWk9jWA= github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9/go.mod h1:2lmrmq866uF2tnje75wQHzmPXhmSWUt7Gyx2vgK1RCU= github.com/drakkan/ftpserverlib v0.0.0-20211107071448-34ff70e85dfb h1:cT/w4XStm7m022JgVqmrXZLcZ4UjoUER1VW5/5gd6ec= @@ -258,7 +260,6 @@ github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHqu github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -414,13 +415,13 @@ github.com/grandcat/zeroconf v1.0.0 h1:uHhahLBKqwWBV6WZUDAT71044vwOTL+McW0mBJvo6 github.com/grandcat/zeroconf v1.0.0/go.mod h1:lTKmG1zh86XyCoUeIHSA4FJMBwCJiQmGfcP2PdzytEs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/api v1.8.1/go.mod h1:sDjTOq0yUyv5G4h+BqSea7Fn6BU+XbolEz1952UB+mk= -github.com/hashicorp/consul/api v1.10.1/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/consul/api v1.11.0/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= +github.com/hashicorp/consul/api v1.12.0/go.mod h1:6pVBMo0ebnYdt2S3H87XhekM/HHrUoTD2XXb/VrZVy0= github.com/hashicorp/consul/sdk v0.7.0/go.mod h1:fY08Y9z5SvJqevyZNy6WWPXiG3KwBPAvlcdx16zZ0fM= github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= @@ -430,31 +431,32 @@ github.com/hashicorp/go-hclog v0.14.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39 github.com/hashicorp/go-hclog v1.0.0 h1:bkKf0BeBXcSYa7f5Fyi9gMuQ8gNsxeiNpZjR6VxNZeo= github.com/hashicorp/go-hclog v1.0.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-plugin v1.4.3 h1:DXmvivbWD5qdiBts9TpBC7BYL1Aia5sxbRgQB+v6UZM= github.com/hashicorp/go-plugin v1.4.3/go.mod h1:5fGEH17QVwTTcR0zV7yhDPLLmFX9YSZ38b18Udy6vYQ= +github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-retryablehttp v0.7.0 h1:eu1EI/mbirUgP5C8hVsTNaGZreBDlYiwC1FZWkvQPQ4= github.com/hashicorp/go-retryablehttp v0.7.0/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= +github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87 h1:xixZ2bWeofWV68J+x6AzmKuVM/JWCQwkWm6GW/MUR6I= github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= @@ -524,6 +526,7 @@ github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= @@ -615,6 +618,7 @@ github.com/mhale/smtpd v0.8.0/go.mod h1:MQl+y2hwIEQCXtNhe5+55n0GZOjSmeqORDIXbqUL github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= +github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= github.com/minio/highwayhash v1.0.1/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= @@ -622,27 +626,23 @@ github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/minio/sio v0.3.0 h1:syEFBewzOMOYVzSTFpp1MqpSZk8rUNbz8VIIc+PNzus= github.com/minio/sio v0.3.0/go.mod h1:8b0yPp2avGThviy/+OCJBI6OMpvxoUuiLvE6F1lebhw= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU= github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nats-io/jwt v1.2.2/go.mod h1:/xX356yQA6LuXI9xWW7mZNpxgF2mBmGecH+Fj34sP5Q= @@ -669,7 +669,7 @@ github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT9 github.com/otiai10/mint v1.3.3 h1:7JgpsBaN0uMkyju4tbYHu0mnM55hNKVYLsXmwr15NQI= github.com/otiai10/mint v1.3.3/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= @@ -694,6 +694,7 @@ github.com/pquerna/otp v1.3.0 h1:oJV/SkzR33anKXwQU3Of42rL4wbrffP4uvUf1SvS5Xs= github.com/pquerna/otp v1.3.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= @@ -703,12 +704,14 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= @@ -723,13 +726,13 @@ github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= -github.com/rs/zerolog v1.26.0 h1:ORM4ibhEZeTeQlCojCK2kPz1ogAY4bGs4tD+SaAdGaE= -github.com/rs/zerolog v1.26.0/go.mod h1:yBiM87lvSqX8h0Ww4sdzNSkVYZ8dL2xjZJG1lAuGZEo= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/rs/zerolog v1.26.1 h1:/ihwxqH+4z8UxyI70wM1z9yCvkWcfz/a3mj48k/Zngc= +github.com/rs/zerolog v1.26.1/go.mod h1:/wSSJWX7lVrsOwlbyTRSOJvqRlc+WjWlfes+CiJ+tmc= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sagikazarmark/crypt v0.1.0/go.mod h1:B/mN0msZuINBtQ1zZLEQcegFJJf9vnYIR88KRMEuODE= +github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= +github.com/sagikazarmark/crypt v0.4.0/go.mod h1:ALv2SRj7GxYV4HO9elxH9nS6M9gW+xDNxqmyJ6RfDFM= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo= @@ -739,7 +742,6 @@ github.com/shirou/gopsutil/v3 v3.21.11/go.mod h1:BToYZVTlSVlfazpDDYFnsVZLaoRG+g8 github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -752,18 +754,17 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= -github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= +github.com/spf13/cobra v1.3.0 h1:R7cSvGu+Vv+qX0gW5R/85dx2kmmJT5z5NM8ifdYjdn0= +github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi9V4= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= -github.com/spf13/viper v1.9.0 h1:yR6EXjTp0y0cLN8OZg1CRZmOBdI88UcGkhgyJhu6nZk= -github.com/spf13/viper v1.9.0/go.mod h1:+i6ajR7OX2XaiBkrcZJFK21htRk7eDeLg7+O6bhUPP4= +github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= +github.com/spf13/viper v1.10.1 h1:nuJZuYpG7gTj/XqiUwg8bA0cp1+M2mC3J4g5luUYBKk= +github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/amqp v1.0.0/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/handy v0.0.0-20200128134331-0f66f006fb2e/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= @@ -785,6 +786,7 @@ github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ= github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8= +github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/wagslane/go-password-validator v0.3.0 h1:vfxOPzGHkz5S146HDpavl0cw1DSVP061Ry2PX0/ON6I= @@ -805,8 +807,11 @@ github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxt go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= +go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0= go.opencensus.io v0.15.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= @@ -887,7 +892,6 @@ golang.org/x/oauth2 v0.0.0-20210126194326-f9ce19ea3013/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210427180440-81ed05c6b58c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= @@ -930,7 +934,6 @@ golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -989,11 +992,13 @@ golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211210111614-af8b64212486 h1:5hpz5aRr+W1erYCL5JRhSUBJRph7l9XkNveoExlrKYk= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1036,7 +1041,6 @@ golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1109,7 +1113,6 @@ google.golang.org/api v0.37.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjR google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= google.golang.org/api v0.46.0/go.mod h1:ceL4oozhkAiTID8XMmJBsIxID/9wMXJVVFXPg4ylg3I= google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= @@ -1121,9 +1124,11 @@ google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqiv google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= google.golang.org/api v0.58.0/go.mod h1:cAbP2FsxoGVNwtgNAmmn3y5G1TWAiVYRmg4yku3lv+E= +google.golang.org/api v0.59.0/go.mod h1:sT2boj7M9YJxZzgeZqXogmhfmRWDtPzT31xkieUbuZU= google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= -google.golang.org/api v0.62.0 h1:PhGymJMXfGBzc4lBRmrx9+1w4w2wEzURHNGF/sD/xGc= google.golang.org/api v0.62.0/go.mod h1:dKmwPCydfsad4qCH08MSdgWjfHOyfpd4VtDGgRFdavw= +google.golang.org/api v0.63.0 h1:n2bqqK895ygnBpdPDYetfy23K7fJ22wsrZKCyfuRkkA= +google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1197,7 +1202,9 @@ google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEc google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211008145708-270636b82663/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211016002631-37fc39342514/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211028162531-8db9c33dc351/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211203200212-54befc351ae9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= @@ -1231,8 +1238,9 @@ google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.42.0 h1:XT2/MFpuPFsEX2fWh3YQtHkZ+WYZFQRfaUgLZYj/p6A= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.43.0 h1:Eeu7bZtDZ2DpRCsLhUlcrLnvYaMK1Gz86a+hMVvELmM= +google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1260,8 +1268,6 @@ gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v9 v9.29.1/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= -gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.63.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= diff --git a/httpd/api_http_user.go b/httpd/api_http_user.go index 73e5ca85..a7408459 100644 --- a/httpd/api_http_user.go +++ b/httpd/api_http_user.go @@ -355,7 +355,7 @@ func deleteUserFile(w http.ResponseWriter, r *http.Request) { var fi os.FileInfo if fi, err = fs.Lstat(p); err != nil { - connection.Log(logger.LevelWarn, "failed to remove a file %#v: stat error: %+v", p, err) + connection.Log(logger.LevelWarn, "failed to remove file %#v: stat error: %+v", p, err) err = connection.GetFsError(fs, err) sendAPIResponse(w, r, err, fmt.Sprintf("Unable to delete file %#v", name), getMappedStatusCode(err)) return diff --git a/httpd/api_metadata.go b/httpd/api_metadata.go new file mode 100644 index 00000000..303af531 --- /dev/null +++ b/httpd/api_metadata.go @@ -0,0 +1,110 @@ +package httpd + +import ( + "fmt" + "net/http" + "sync" + "time" + + "github.com/go-chi/render" + + "github.com/drakkan/sftpgo/v2/dataprovider" + "github.com/drakkan/sftpgo/v2/logger" + "github.com/drakkan/sftpgo/v2/util" +) + +var ( + activeMetadataChecks metadataChecks +) + +type metadataCheck struct { + // Username to which the metadata check refers + Username string `json:"username"` + // check start time as unix timestamp in milliseconds + StartTime int64 `json:"start_time"` +} + +// metadataChecks holds the active metadata checks +type metadataChecks struct { + sync.RWMutex + checks []metadataCheck +} + +func (c *metadataChecks) get() []metadataCheck { + c.RLock() + defer c.RUnlock() + + checks := make([]metadataCheck, len(c.checks)) + copy(checks, c.checks) + + return checks +} + +func (c *metadataChecks) add(username string) bool { + c.Lock() + defer c.Unlock() + + for idx := range c.checks { + if c.checks[idx].Username == username { + return false + } + } + + c.checks = append(c.checks, metadataCheck{ + Username: username, + StartTime: util.GetTimeAsMsSinceEpoch(time.Now()), + }) + + return true +} + +func (c *metadataChecks) remove(username string) bool { + c.Lock() + defer c.Unlock() + + for idx := range c.checks { + if c.checks[idx].Username == username { + lastIdx := len(c.checks) - 1 + c.checks[idx] = c.checks[lastIdx] + c.checks = c.checks[:lastIdx] + return true + } + } + + return false +} + +func getMetadataChecks(w http.ResponseWriter, r *http.Request) { + r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize) + render.JSON(w, r, activeMetadataChecks.get()) +} + +func startMetadataCheck(w http.ResponseWriter, r *http.Request) { + r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize) + + user, err := dataprovider.UserExists(getURLParam(r, "username")) + if err != nil { + sendAPIResponse(w, r, err, "", getRespStatus(err)) + return + } + if !activeMetadataChecks.add(user.Username) { + sendAPIResponse(w, r, err, fmt.Sprintf("Another check is already in progress for user %#v", user.Username), + http.StatusConflict) + return + } + go doMetadataCheck(user) //nolint:errcheck + + sendAPIResponse(w, r, err, "Check started", http.StatusAccepted) +} + +func doMetadataCheck(user dataprovider.User) error { + defer activeMetadataChecks.remove(user.Username) + + err := user.CheckMetadataConsistency() + if err != nil { + logger.Warn(logSender, "", "error checking metadata for user %#v: %v", user.Username, err) + return err + } + logger.Debug(logSender, "", "metadata check completed for user: %#v", user.Username) + return nil +} diff --git a/httpd/httpd.go b/httpd/httpd.go index d633dc41..d7247f9c 100644 --- a/httpd/httpd.go +++ b/httpd/httpd.go @@ -79,6 +79,8 @@ const ( userSharesPath = "/api/v2/user/shares" retentionBasePath = "/api/v2/retention/users" retentionChecksPath = "/api/v2/retention/users/checks" + metadataBasePath = "/api/v2/metadata/users" + metadataChecksPath = "/api/v2/metadata/users/checks" fsEventsPath = "/api/v2/events/fs" providerEventsPath = "/api/v2/events/provider" sharesPath = "/api/v2/shares" diff --git a/httpd/httpd_test.go b/httpd/httpd_test.go index b916069c..e1408485 100644 --- a/httpd/httpd_test.go +++ b/httpd/httpd_test.go @@ -108,6 +108,7 @@ const ( userProfilePath = "/api/v2/user/profile" userSharesPath = "/api/v2/user/shares" retentionBasePath = "/api/v2/retention/users" + metadataBasePath = "/api/v2/metadata/users" fsEventsPath = "/api/v2/events/fs" providerEventsPath = "/api/v2/events/provider" sharesPath = "/api/v2/shares" @@ -1699,6 +1700,52 @@ func TestUserType(t *testing.T) { assert.NoError(t, err) } +func TestMetadataAPI(t *testing.T) { + user, _, err := httpdtest.AddUser(getTestUser(), http.StatusCreated) + assert.NoError(t, err) + + token, err := getJWTAPITokenFromTestServer(defaultTokenAuthUser, defaultTokenAuthPass) + assert.NoError(t, err) + req, err := http.NewRequest(http.MethodGet, path.Join(metadataBasePath, "/checks"), nil) + assert.NoError(t, err) + setBearerForReq(req, token) + rr := executeRequest(req) + checkResponseCode(t, http.StatusOK, rr) + var resp []interface{} + err = json.Unmarshal(rr.Body.Bytes(), &resp) + assert.NoError(t, err) + assert.Len(t, resp, 0) + + req, err = http.NewRequest(http.MethodPost, path.Join(metadataBasePath, user.Username, "/check"), nil) + assert.NoError(t, err) + setBearerForReq(req, token) + rr = executeRequest(req) + checkResponseCode(t, http.StatusAccepted, rr) + + assert.Eventually(t, func() bool { + req, err := http.NewRequest(http.MethodGet, path.Join(metadataBasePath, "/checks"), nil) + assert.NoError(t, err) + setBearerForReq(req, token) + rr := executeRequest(req) + checkResponseCode(t, http.StatusOK, rr) + var resp []interface{} + err = json.Unmarshal(rr.Body.Bytes(), &resp) + assert.NoError(t, err) + return len(resp) == 0 + }, 1000*time.Millisecond, 50*time.Millisecond) + + _, err = httpdtest.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) + + req, err = http.NewRequest(http.MethodPost, path.Join(metadataBasePath, user.Username, "/check"), nil) + assert.NoError(t, err) + setBearerForReq(req, token) + rr = executeRequest(req) + checkResponseCode(t, http.StatusNotFound, rr) +} + func TestRetentionAPI(t *testing.T) { user, _, err := httpdtest.AddUser(getTestUser(), http.StatusCreated) assert.NoError(t, err) diff --git a/httpd/internal_test.go b/httpd/internal_test.go index f33c243b..81326adb 100644 --- a/httpd/internal_test.go +++ b/httpd/internal_test.go @@ -2134,3 +2134,42 @@ func TestUserCanResetPassword(t *testing.T) { u.Filters.AllowedIP = []string{"127.0.0.1/8"} assert.False(t, isUserAllowedToResetPassword(req, &u)) } + +func TestMetadataAPI(t *testing.T) { + username := "metadatauser" + assert.False(t, activeMetadataChecks.remove(username)) + + user := dataprovider.User{ + BaseUser: sdk.BaseUser{ + Username: username, + Password: "metadata_pwd", + HomeDir: filepath.Join(os.TempDir(), username), + Status: 1, + }, + } + user.Permissions = make(map[string][]string) + user.Permissions["/"] = []string{dataprovider.PermAny} + err := dataprovider.AddUser(&user, "", "") + assert.NoError(t, err) + + assert.True(t, activeMetadataChecks.add(username)) + + req, err := http.NewRequest(http.MethodPost, path.Join(metadataBasePath, username, "check"), nil) + assert.NoError(t, err) + rctx := chi.NewRouteContext() + rctx.URLParams.Add("username", username) + req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx)) + + rr := httptest.NewRecorder() + startMetadataCheck(rr, req) + assert.Equal(t, http.StatusConflict, rr.Code) + + assert.True(t, activeMetadataChecks.remove(username)) + assert.Len(t, activeMetadataChecks.get(), 0) + err = dataprovider.DeleteUser(username, "", "") + assert.NoError(t, err) + + user.FsConfig.Provider = sdk.AzureBlobFilesystemProvider + err = doMetadataCheck(user) + assert.Error(t, err) +} diff --git a/httpd/server.go b/httpd/server.go index b9d554a9..0f2c1371 100644 --- a/httpd/server.go +++ b/httpd/server.go @@ -1093,6 +1093,9 @@ func (s *httpdServer) initializeRouter() { router.With(checkPerm(dataprovider.PermAdminRetentionChecks)).Get(retentionChecksPath, getRetentionChecks) router.With(checkPerm(dataprovider.PermAdminRetentionChecks)).Post(retentionBasePath+"/{username}/check", startRetentionCheck) + router.With(checkPerm(dataprovider.PermAdminMetadataChecks)).Get(metadataChecksPath, getMetadataChecks) + router.With(checkPerm(dataprovider.PermAdminMetadataChecks)).Post(metadataBasePath+"/{username}/check", + startMetadataCheck) router.With(checkPerm(dataprovider.PermAdminViewEvents), compressor.Handler). Get(fsEventsPath, searchFsEvents) router.With(checkPerm(dataprovider.PermAdminViewEvents), compressor.Handler). diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index 13a487b6..7942a9e1 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -12,6 +12,7 @@ tags: - name: users - name: data retention - name: events + - name: metadata - name: user APIs - name: public shares info: @@ -898,6 +899,67 @@ paths: $ref: '#/components/responses/InternalServerError' default: $ref: '#/components/responses/DefaultResponse' + /metadata/users/checks: + get: + tags: + - metadata + summary: Get metadata checks + description: Returns the active metadata checks + operationId: get_users_metadata_checks + responses: + '200': + description: successful operation + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/MetadataCheck' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + '500': + $ref: '#/components/responses/InternalServerError' + default: + $ref: '#/components/responses/DefaultResponse' + /metadata/users/{username}/check: + parameters: + - name: username + in: path + description: the username + required: true + schema: + type: string + post: + tags: + - metadata + summary: Start a metadata check + description: 'Starts a new metadata check for the given user. A metadata check requires a metadata plugin and removes the metadata associated to missing items (for example objects deleted outside SFTPGo). If a metadata check for this user is already active a 409 status code is returned. Metadata are stored for cloud storage backends. This API does nothing for other backends or if no metadata plugin is configured' + operationId: start_user_metadata_check + responses: + '202': + description: successful operation + content: + application/json: + schema: + $ref: '#/components/schemas/ApiResponse' + example: + message: Check started + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + '404': + $ref: '#/components/responses/NotFound' + '409': + $ref: '#/components/responses/Conflict' + '500': + $ref: '#/components/responses/InternalServerError' + default: + $ref: '#/components/responses/DefaultResponse' /retention/users/checks: get: tags: @@ -4029,6 +4091,7 @@ components: - manage_defender - view_defender - retention_checks + - metadata_checks - view_events description: | Admin permissions: @@ -4047,6 +4110,7 @@ components: * `manage_defender` - remove ip from the dynamic blocklist is allowed * `view_defender` - list the dynamic blocklist is allowed * `retention_checks` - view and start retention checks is allowed + * `metadata_checks` - view and start metadata checks is allowed * `view_events` - view and search filesystem and provider events is allowed LoginMethods: type: string @@ -4984,6 +5048,16 @@ components: type: string format: email description: 'if the notification method is set to "Email", this is the e-mail address that receives the retention check report. This field is automatically set to the email address associated with the administrator starting the check' + MetadataCheck: + type: object + properties: + username: + type: string + description: username to which the check refers + start_time: + type: integer + format: int64 + description: check start time as unix timestamp in milliseconds QuotaScan: type: object properties: diff --git a/pkgs/build.sh b/pkgs/build.sh index 3d7bb44d..99f3832a 100755 --- a/pkgs/build.sh +++ b/pkgs/build.sh @@ -1,6 +1,6 @@ #!/bin/bash -NFPM_VERSION=2.10.0 +NFPM_VERSION=2.11.0 NFPM_ARCH=${NFPM_ARCH:-amd64} if [ -z ${SFTPGO_VERSION} ] then diff --git a/sdk/plugin/eventsearcher/eventsearcher.go b/sdk/plugin/eventsearcher/eventsearcher.go index eed4d8c7..256169a3 100644 --- a/sdk/plugin/eventsearcher/eventsearcher.go +++ b/sdk/plugin/eventsearcher/eventsearcher.go @@ -36,7 +36,7 @@ type Searcher interface { limit, order int, actions, objectTypes, instanceIDs, excludeIDs []string) ([]byte, []string, []string, error) } -// Plugin defines the implementation to serve/connect to a notifier plugin +// Plugin defines the implementation to serve/connect to a event search plugin type Plugin struct { plugin.Plugin Impl Searcher diff --git a/sdk/plugin/eventsearcher/grpc.go b/sdk/plugin/eventsearcher/grpc.go index 441b2ef5..e9713213 100644 --- a/sdk/plugin/eventsearcher/grpc.go +++ b/sdk/plugin/eventsearcher/grpc.go @@ -76,7 +76,7 @@ type GRPCServer struct { Impl Searcher } -// SearchFsEvents implement the server side fs events search method +// SearchFsEvents implements the server side fs events search method func (s *GRPCServer) SearchFsEvents(ctx context.Context, req *proto.FsEventsFilter) (*proto.SearchResponse, error) { responseData, sameTsAtStart, sameTsAtEnd, err := s.Impl.SearchFsEvents(req.StartTimestamp, req.EndTimestamp, req.Username, req.Ip, req.SshCmd, req.Actions, req.Protocols, req.InstanceIds, diff --git a/sdk/plugin/metadata.go b/sdk/plugin/metadata.go new file mode 100644 index 00000000..dae88724 --- /dev/null +++ b/sdk/plugin/metadata.go @@ -0,0 +1,82 @@ +package plugin + +import ( + "crypto/sha256" + "fmt" + "os/exec" + + "github.com/hashicorp/go-hclog" + "github.com/hashicorp/go-plugin" + + "github.com/drakkan/sftpgo/v2/logger" + "github.com/drakkan/sftpgo/v2/sdk/plugin/metadata" +) + +type metadataPlugin struct { + config Config + metadater metadata.Metadater + client *plugin.Client +} + +func newMetadaterPlugin(config Config) (*metadataPlugin, error) { + p := &metadataPlugin{ + config: config, + } + if err := p.initialize(); err != nil { + logger.Warn(logSender, "", "unable to create metadata plugin: %v, config %+v", err, config) + return nil, err + } + return p, nil +} + +func (p *metadataPlugin) exited() bool { + return p.client.Exited() +} + +func (p *metadataPlugin) cleanup() { + p.client.Kill() +} + +func (p *metadataPlugin) initialize() error { + killProcess(p.config.Cmd) + logger.Debug(logSender, "", "create new metadata plugin %#v", p.config.Cmd) + var secureConfig *plugin.SecureConfig + if p.config.SHA256Sum != "" { + secureConfig.Checksum = []byte(p.config.SHA256Sum) + secureConfig.Hash = sha256.New() + } + client := plugin.NewClient(&plugin.ClientConfig{ + HandshakeConfig: metadata.Handshake, + Plugins: metadata.PluginMap, + Cmd: exec.Command(p.config.Cmd, p.config.Args...), + AllowedProtocols: []plugin.Protocol{ + plugin.ProtocolGRPC, + }, + Managed: false, + AutoMTLS: p.config.AutoMTLS, + SecureConfig: secureConfig, + Logger: &logger.HCLogAdapter{ + Logger: hclog.New(&hclog.LoggerOptions{ + Name: fmt.Sprintf("%v.%v", logSender, metadata.PluginName), + Level: pluginsLogLevel, + DisableTime: true, + }), + }, + }) + rpcClient, err := client.Client() + if err != nil { + logger.Debug(logSender, "", "unable to get rpc client for plugin %#v: %v", p.config.Cmd, err) + return err + } + raw, err := rpcClient.Dispense(metadata.PluginName) + if err != nil { + logger.Debug(logSender, "", "unable to get plugin %v from rpc client for command %#v: %v", + metadata.PluginName, p.config.Cmd, err) + return err + } + + p.client = client + p.metadater = raw.(metadata.Metadater) + + return nil +} diff --git a/sdk/plugin/metadata/grpc.go b/sdk/plugin/metadata/grpc.go new file mode 100644 index 00000000..023747ca --- /dev/null +++ b/sdk/plugin/metadata/grpc.go @@ -0,0 +1,160 @@ +package metadata + +import ( + "context" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/emptypb" + + "github.com/drakkan/sftpgo/v2/sdk/plugin/metadata/proto" +) + +const ( + rpcTimeout = 20 * time.Second +) + +// GRPCClient is an implementation of Metadater interface that talks over RPC. +type GRPCClient struct { + client proto.MetadataClient +} + +// SetModificationTime implements the Metadater interface +func (c *GRPCClient) SetModificationTime(storageID, objectPath string, mTime int64) error { + ctx, cancel := context.WithTimeout(context.Background(), rpcTimeout) + defer cancel() + + _, err := c.client.SetModificationTime(ctx, &proto.SetModificationTimeRequest{ + StorageId: storageID, + ObjectPath: objectPath, + ModificationTime: mTime, + }) + + return c.checkError(err) +} + +// GetModificationTime implements the Metadater interface +func (c *GRPCClient) GetModificationTime(storageID, objectPath string) (int64, error) { + ctx, cancel := context.WithTimeout(context.Background(), rpcTimeout) + defer cancel() + + resp, err := c.client.GetModificationTime(ctx, &proto.GetModificationTimeRequest{ + StorageId: storageID, + ObjectPath: objectPath, + }) + + if err != nil { + return 0, c.checkError(err) + } + + return resp.ModificationTime, nil +} + +// GetModificationTimes implements the Metadater interface +func (c *GRPCClient) GetModificationTimes(storageID, objectPath string) (map[string]int64, error) { + ctx, cancel := context.WithTimeout(context.Background(), rpcTimeout*4) + defer cancel() + + resp, err := c.client.GetModificationTimes(ctx, &proto.GetModificationTimesRequest{ + StorageId: storageID, + FolderPath: objectPath, + }) + + if err != nil { + return nil, c.checkError(err) + } + + return resp.Pairs, nil +} + +// RemoveMetadata implements the Metadater interface +func (c *GRPCClient) RemoveMetadata(storageID, objectPath string) error { + ctx, cancel := context.WithTimeout(context.Background(), rpcTimeout) + defer cancel() + + _, err := c.client.RemoveMetadata(ctx, &proto.RemoveMetadataRequest{ + StorageId: storageID, + ObjectPath: objectPath, + }) + + return c.checkError(err) +} + +// GetFolders implements the Metadater interface +func (c *GRPCClient) GetFolders(storageID string, limit int, from string) ([]string, error) { + ctx, cancel := context.WithTimeout(context.Background(), rpcTimeout) + defer cancel() + + resp, err := c.client.GetFolders(ctx, &proto.GetFoldersRequest{ + StorageId: storageID, + Limit: int32(limit), + From: from, + }) + if err != nil { + return nil, c.checkError(err) + } + return resp.Folders, nil +} + +func (c *GRPCClient) checkError(err error) error { + if err == nil { + return nil + } + if s, ok := status.FromError(err); ok { + if s.Code() == codes.NotFound { + return ErrNoSuchObject + } + } + return err +} + +// GRPCServer defines the gRPC server that GRPCClient talks to. +type GRPCServer struct { + Impl Metadater +} + +// SetModificationTime implements the server side set modification time method +func (s *GRPCServer) SetModificationTime(ctx context.Context, req *proto.SetModificationTimeRequest) (*emptypb.Empty, error) { + err := s.Impl.SetModificationTime(req.StorageId, req.ObjectPath, req.ModificationTime) + + return &emptypb.Empty{}, err +} + +// GetModificationTime implements the server side get modification time method +func (s *GRPCServer) GetModificationTime(ctx context.Context, req *proto.GetModificationTimeRequest) ( + *proto.GetModificationTimeResponse, error, +) { + mTime, err := s.Impl.GetModificationTime(req.StorageId, req.ObjectPath) + + return &proto.GetModificationTimeResponse{ + ModificationTime: mTime, + }, err +} + +// GetModificationTimes implements the server side get modification times method +func (s *GRPCServer) GetModificationTimes(ctx context.Context, req *proto.GetModificationTimesRequest) ( + *proto.GetModificationTimesResponse, error, +) { + res, err := s.Impl.GetModificationTimes(req.StorageId, req.FolderPath) + + return &proto.GetModificationTimesResponse{ + Pairs: res, + }, err +} + +// RemoveMetadata implements the server side remove metadata method +func (s *GRPCServer) RemoveMetadata(ctx context.Context, req *proto.RemoveMetadataRequest) (*emptypb.Empty, error) { + err := s.Impl.RemoveMetadata(req.StorageId, req.ObjectPath) + + return &emptypb.Empty{}, err +} + +// GetFolders implements the server side get folders method +func (s *GRPCServer) GetFolders(ctx context.Context, req *proto.GetFoldersRequest) (*proto.GetFoldersResponse, error) { + res, err := s.Impl.GetFolders(req.StorageId, int(req.Limit), req.From) + + return &proto.GetFoldersResponse{ + Folders: res, + }, err +} diff --git a/sdk/plugin/metadata/metadata.go b/sdk/plugin/metadata/metadata.go new file mode 100644 index 00000000..8037811d --- /dev/null +++ b/sdk/plugin/metadata/metadata.go @@ -0,0 +1,61 @@ +package metadata + +import ( + "context" + "errors" + + "github.com/hashicorp/go-plugin" + "google.golang.org/grpc" + + "github.com/drakkan/sftpgo/v2/sdk/plugin/metadata/proto" +) + +const ( + // PluginName defines the name for a metadata plugin + PluginName = "metadata" +) + +var ( + // Handshake is a common handshake that is shared by plugin and host. + Handshake = plugin.HandshakeConfig{ + ProtocolVersion: 1, + MagicCookieKey: "SFTPGO_PLUGIN_METADATA", + MagicCookieValue: "85dddeea-56d8-4d5b-b488-8b125edb3a0f", + } + // ErrNoSuchObject is the error that plugins must return if the request object does not exist + ErrNoSuchObject = errors.New("no such object") + // PluginMap is the map of plugins we can dispense. + PluginMap = map[string]plugin.Plugin{ + PluginName: &Plugin{}, + } +) + +// Metadater defines the interface for metadata plugins +type Metadater interface { + SetModificationTime(storageID, objectPath string, mTime int64) error + GetModificationTime(storageID, objectPath string) (int64, error) + GetModificationTimes(storageID, objectPath string) (map[string]int64, error) + RemoveMetadata(storageID, objectPath string) error + GetFolders(storageID string, limit int, from string) ([]string, error) +} + +// Plugin defines the implementation to serve/connect to a metadata plugin +type Plugin struct { + plugin.Plugin + Impl Metadater +} + +// GRPCServer defines the GRPC server implementation for this plugin +func (p *Plugin) GRPCServer(broker *plugin.GRPCBroker, s *grpc.Server) error { + proto.RegisterMetadataServer(s, &GRPCServer{ + Impl: p.Impl, + }) + return nil +} + +// GRPCClient defines the GRPC client implementation for this plugin +func (p *Plugin) GRPCClient(ctx context.Context, broker *plugin.GRPCBroker, c *grpc.ClientConn) (interface{}, error) { + return &GRPCClient{ + client: proto.NewMetadataClient(c), + }, nil +} diff --git a/sdk/plugin/metadata/proto/metadata.pb.go b/sdk/plugin/metadata/proto/metadata.pb.go new file mode 100644 index 00000000..6b3f1d7b --- /dev/null +++ b/sdk/plugin/metadata/proto/metadata.pb.go @@ -0,0 +1,938 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.26.0 +// protoc v3.17.3 +// source: metadata/proto/metadata.proto + +package proto + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + emptypb "google.golang.org/protobuf/types/known/emptypb" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type SetModificationTimeRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + StorageId string `protobuf:"bytes,1,opt,name=storage_id,json=storageId,proto3" json:"storage_id,omitempty"` + ObjectPath string `protobuf:"bytes,2,opt,name=object_path,json=objectPath,proto3" json:"object_path,omitempty"` + ModificationTime int64 `protobuf:"varint,3,opt,name=modification_time,json=modificationTime,proto3" json:"modification_time,omitempty"` +} + +func (x *SetModificationTimeRequest) Reset() { + *x = SetModificationTimeRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_metadata_proto_metadata_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SetModificationTimeRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SetModificationTimeRequest) ProtoMessage() {} + +func (x *SetModificationTimeRequest) ProtoReflect() protoreflect.Message { + mi := &file_metadata_proto_metadata_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SetModificationTimeRequest.ProtoReflect.Descriptor instead. +func (*SetModificationTimeRequest) Descriptor() ([]byte, []int) { + return file_metadata_proto_metadata_proto_rawDescGZIP(), []int{0} +} + +func (x *SetModificationTimeRequest) GetStorageId() string { + if x != nil { + return x.StorageId + } + return "" +} + +func (x *SetModificationTimeRequest) GetObjectPath() string { + if x != nil { + return x.ObjectPath + } + return "" +} + +func (x *SetModificationTimeRequest) GetModificationTime() int64 { + if x != nil { + return x.ModificationTime + } + return 0 +} + +type GetModificationTimeRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + StorageId string `protobuf:"bytes,1,opt,name=storage_id,json=storageId,proto3" json:"storage_id,omitempty"` + ObjectPath string `protobuf:"bytes,2,opt,name=object_path,json=objectPath,proto3" json:"object_path,omitempty"` +} + +func (x *GetModificationTimeRequest) Reset() { + *x = GetModificationTimeRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_metadata_proto_metadata_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetModificationTimeRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetModificationTimeRequest) ProtoMessage() {} + +func (x *GetModificationTimeRequest) ProtoReflect() protoreflect.Message { + mi := &file_metadata_proto_metadata_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetModificationTimeRequest.ProtoReflect.Descriptor instead. +func (*GetModificationTimeRequest) Descriptor() ([]byte, []int) { + return file_metadata_proto_metadata_proto_rawDescGZIP(), []int{1} +} + +func (x *GetModificationTimeRequest) GetStorageId() string { + if x != nil { + return x.StorageId + } + return "" +} + +func (x *GetModificationTimeRequest) GetObjectPath() string { + if x != nil { + return x.ObjectPath + } + return "" +} + +type GetModificationTimeResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ModificationTime int64 `protobuf:"varint,1,opt,name=modification_time,json=modificationTime,proto3" json:"modification_time,omitempty"` +} + +func (x *GetModificationTimeResponse) Reset() { + *x = GetModificationTimeResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_metadata_proto_metadata_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetModificationTimeResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetModificationTimeResponse) ProtoMessage() {} + +func (x *GetModificationTimeResponse) ProtoReflect() protoreflect.Message { + mi := &file_metadata_proto_metadata_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetModificationTimeResponse.ProtoReflect.Descriptor instead. +func (*GetModificationTimeResponse) Descriptor() ([]byte, []int) { + return file_metadata_proto_metadata_proto_rawDescGZIP(), []int{2} +} + +func (x *GetModificationTimeResponse) GetModificationTime() int64 { + if x != nil { + return x.ModificationTime + } + return 0 +} + +type GetModificationTimesRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + StorageId string `protobuf:"bytes,1,opt,name=storage_id,json=storageId,proto3" json:"storage_id,omitempty"` + FolderPath string `protobuf:"bytes,2,opt,name=folder_path,json=folderPath,proto3" json:"folder_path,omitempty"` +} + +func (x *GetModificationTimesRequest) Reset() { + *x = GetModificationTimesRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_metadata_proto_metadata_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetModificationTimesRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetModificationTimesRequest) ProtoMessage() {} + +func (x *GetModificationTimesRequest) ProtoReflect() protoreflect.Message { + mi := &file_metadata_proto_metadata_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetModificationTimesRequest.ProtoReflect.Descriptor instead. +func (*GetModificationTimesRequest) Descriptor() ([]byte, []int) { + return file_metadata_proto_metadata_proto_rawDescGZIP(), []int{3} +} + +func (x *GetModificationTimesRequest) GetStorageId() string { + if x != nil { + return x.StorageId + } + return "" +} + +func (x *GetModificationTimesRequest) GetFolderPath() string { + if x != nil { + return x.FolderPath + } + return "" +} + +type GetModificationTimesResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // the file name (not the full path) is the map key and the modification time is the map value + Pairs map[string]int64 `protobuf:"bytes,1,rep,name=pairs,proto3" json:"pairs,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` +} + +func (x *GetModificationTimesResponse) Reset() { + *x = GetModificationTimesResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_metadata_proto_metadata_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetModificationTimesResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetModificationTimesResponse) ProtoMessage() {} + +func (x *GetModificationTimesResponse) ProtoReflect() protoreflect.Message { + mi := &file_metadata_proto_metadata_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetModificationTimesResponse.ProtoReflect.Descriptor instead. +func (*GetModificationTimesResponse) Descriptor() ([]byte, []int) { + return file_metadata_proto_metadata_proto_rawDescGZIP(), []int{4} +} + +func (x *GetModificationTimesResponse) GetPairs() map[string]int64 { + if x != nil { + return x.Pairs + } + return nil +} + +type RemoveMetadataRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + StorageId string `protobuf:"bytes,1,opt,name=storage_id,json=storageId,proto3" json:"storage_id,omitempty"` + ObjectPath string `protobuf:"bytes,2,opt,name=object_path,json=objectPath,proto3" json:"object_path,omitempty"` +} + +func (x *RemoveMetadataRequest) Reset() { + *x = RemoveMetadataRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_metadata_proto_metadata_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RemoveMetadataRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RemoveMetadataRequest) ProtoMessage() {} + +func (x *RemoveMetadataRequest) ProtoReflect() protoreflect.Message { + mi := &file_metadata_proto_metadata_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RemoveMetadataRequest.ProtoReflect.Descriptor instead. +func (*RemoveMetadataRequest) Descriptor() ([]byte, []int) { + return file_metadata_proto_metadata_proto_rawDescGZIP(), []int{5} +} + +func (x *RemoveMetadataRequest) GetStorageId() string { + if x != nil { + return x.StorageId + } + return "" +} + +func (x *RemoveMetadataRequest) GetObjectPath() string { + if x != nil { + return x.ObjectPath + } + return "" +} + +type GetFoldersRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + StorageId string `protobuf:"bytes,1,opt,name=storage_id,json=storageId,proto3" json:"storage_id,omitempty"` + Limit int32 `protobuf:"varint,2,opt,name=limit,proto3" json:"limit,omitempty"` + From string `protobuf:"bytes,3,opt,name=from,proto3" json:"from,omitempty"` +} + +func (x *GetFoldersRequest) Reset() { + *x = GetFoldersRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_metadata_proto_metadata_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetFoldersRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetFoldersRequest) ProtoMessage() {} + +func (x *GetFoldersRequest) ProtoReflect() protoreflect.Message { + mi := &file_metadata_proto_metadata_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetFoldersRequest.ProtoReflect.Descriptor instead. +func (*GetFoldersRequest) Descriptor() ([]byte, []int) { + return file_metadata_proto_metadata_proto_rawDescGZIP(), []int{6} +} + +func (x *GetFoldersRequest) GetStorageId() string { + if x != nil { + return x.StorageId + } + return "" +} + +func (x *GetFoldersRequest) GetLimit() int32 { + if x != nil { + return x.Limit + } + return 0 +} + +func (x *GetFoldersRequest) GetFrom() string { + if x != nil { + return x.From + } + return "" +} + +type GetFoldersResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Folders []string `protobuf:"bytes,1,rep,name=folders,proto3" json:"folders,omitempty"` +} + +func (x *GetFoldersResponse) Reset() { + *x = GetFoldersResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_metadata_proto_metadata_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetFoldersResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetFoldersResponse) ProtoMessage() {} + +func (x *GetFoldersResponse) ProtoReflect() protoreflect.Message { + mi := &file_metadata_proto_metadata_proto_msgTypes[7] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetFoldersResponse.ProtoReflect.Descriptor instead. +func (*GetFoldersResponse) Descriptor() ([]byte, []int) { + return file_metadata_proto_metadata_proto_rawDescGZIP(), []int{7} +} + +func (x *GetFoldersResponse) GetFolders() []string { + if x != nil { + return x.Folders + } + return nil +} + +var File_metadata_proto_metadata_proto protoreflect.FileDescriptor + +var file_metadata_proto_metadata_proto_rawDesc = []byte{ + 0x0a, 0x1d, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x2f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, + 0x05, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x22, 0x89, 0x01, 0x0a, 0x1a, 0x53, 0x65, 0x74, 0x4d, 0x6f, 0x64, 0x69, 0x66, + 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x49, + 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x70, 0x61, 0x74, 0x68, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x50, 0x61, + 0x74, 0x68, 0x12, 0x2b, 0x0a, 0x11, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x10, 0x6d, + 0x6f, 0x64, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x22, + 0x5c, 0x0a, 0x1a, 0x47, 0x65, 0x74, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, + 0x0a, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x09, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x49, 0x64, 0x12, 0x1f, 0x0a, 0x0b, + 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0a, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x50, 0x61, 0x74, 0x68, 0x22, 0x4a, 0x0a, + 0x1b, 0x47, 0x65, 0x74, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x54, 0x69, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2b, 0x0a, 0x11, + 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x69, 0x6d, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x10, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x63, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x22, 0x5d, 0x0a, 0x1b, 0x47, 0x65, 0x74, + 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x69, 0x6d, 0x65, + 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x74, 0x6f, 0x72, + 0x61, 0x67, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x74, + 0x6f, 0x72, 0x61, 0x67, 0x65, 0x49, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x66, 0x6f, 0x6c, 0x64, 0x65, + 0x72, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x66, 0x6f, + 0x6c, 0x64, 0x65, 0x72, 0x50, 0x61, 0x74, 0x68, 0x22, 0x9e, 0x01, 0x0a, 0x1c, 0x47, 0x65, 0x74, + 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x69, 0x6d, 0x65, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x44, 0x0a, 0x05, 0x70, 0x61, 0x69, + 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x54, 0x69, 0x6d, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x50, 0x61, + 0x69, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x05, 0x70, 0x61, 0x69, 0x72, 0x73, 0x1a, + 0x38, 0x0a, 0x0a, 0x50, 0x61, 0x69, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, + 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, + 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x57, 0x0a, 0x15, 0x52, 0x65, 0x6d, + 0x6f, 0x76, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x49, + 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x70, 0x61, 0x74, 0x68, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x50, 0x61, + 0x74, 0x68, 0x22, 0x5c, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x46, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x74, 0x6f, 0x72, 0x61, + 0x67, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x74, 0x6f, + 0x72, 0x61, 0x67, 0x65, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x12, 0x0a, 0x04, + 0x66, 0x72, 0x6f, 0x6d, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x72, 0x6f, 0x6d, + 0x22, 0x2e, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x46, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x66, 0x6f, 0x6c, 0x64, 0x65, 0x72, + 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x66, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x73, + 0x32, 0xa6, 0x03, 0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x50, 0x0a, + 0x13, 0x53, 0x65, 0x74, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x54, 0x69, 0x6d, 0x65, 0x12, 0x21, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x74, + 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x69, 0x6d, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, + 0x5c, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x21, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, + 0x65, 0x74, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x69, + 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5f, 0x0a, + 0x14, 0x47, 0x65, 0x74, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x54, 0x69, 0x6d, 0x65, 0x73, 0x12, 0x22, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, + 0x74, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x69, 0x6d, + 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, + 0x0a, 0x0e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x12, 0x1c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x4d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x41, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x46, 0x6f, 0x6c, + 0x64, 0x65, 0x72, 0x73, 0x12, 0x18, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, + 0x46, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x6f, 0x6c, 0x64, 0x65, 0x72, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x1b, 0x5a, 0x19, 0x73, 0x64, 0x6b, + 0x2f, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x2f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_metadata_proto_metadata_proto_rawDescOnce sync.Once + file_metadata_proto_metadata_proto_rawDescData = file_metadata_proto_metadata_proto_rawDesc +) + +func file_metadata_proto_metadata_proto_rawDescGZIP() []byte { + file_metadata_proto_metadata_proto_rawDescOnce.Do(func() { + file_metadata_proto_metadata_proto_rawDescData = protoimpl.X.CompressGZIP(file_metadata_proto_metadata_proto_rawDescData) + }) + return file_metadata_proto_metadata_proto_rawDescData +} + +var file_metadata_proto_metadata_proto_msgTypes = make([]protoimpl.MessageInfo, 9) +var file_metadata_proto_metadata_proto_goTypes = []interface{}{ + (*SetModificationTimeRequest)(nil), // 0: proto.SetModificationTimeRequest + (*GetModificationTimeRequest)(nil), // 1: proto.GetModificationTimeRequest + (*GetModificationTimeResponse)(nil), // 2: proto.GetModificationTimeResponse + (*GetModificationTimesRequest)(nil), // 3: proto.GetModificationTimesRequest + (*GetModificationTimesResponse)(nil), // 4: proto.GetModificationTimesResponse + (*RemoveMetadataRequest)(nil), // 5: proto.RemoveMetadataRequest + (*GetFoldersRequest)(nil), // 6: proto.GetFoldersRequest + (*GetFoldersResponse)(nil), // 7: proto.GetFoldersResponse + nil, // 8: proto.GetModificationTimesResponse.PairsEntry + (*emptypb.Empty)(nil), // 9: google.protobuf.Empty +} +var file_metadata_proto_metadata_proto_depIdxs = []int32{ + 8, // 0: proto.GetModificationTimesResponse.pairs:type_name -> proto.GetModificationTimesResponse.PairsEntry + 0, // 1: proto.Metadata.SetModificationTime:input_type -> proto.SetModificationTimeRequest + 1, // 2: proto.Metadata.GetModificationTime:input_type -> proto.GetModificationTimeRequest + 3, // 3: proto.Metadata.GetModificationTimes:input_type -> proto.GetModificationTimesRequest + 5, // 4: proto.Metadata.RemoveMetadata:input_type -> proto.RemoveMetadataRequest + 6, // 5: proto.Metadata.GetFolders:input_type -> proto.GetFoldersRequest + 9, // 6: proto.Metadata.SetModificationTime:output_type -> google.protobuf.Empty + 2, // 7: proto.Metadata.GetModificationTime:output_type -> proto.GetModificationTimeResponse + 4, // 8: proto.Metadata.GetModificationTimes:output_type -> proto.GetModificationTimesResponse + 9, // 9: proto.Metadata.RemoveMetadata:output_type -> google.protobuf.Empty + 7, // 10: proto.Metadata.GetFolders:output_type -> proto.GetFoldersResponse + 6, // [6:11] is the sub-list for method output_type + 1, // [1:6] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_metadata_proto_metadata_proto_init() } +func file_metadata_proto_metadata_proto_init() { + if File_metadata_proto_metadata_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_metadata_proto_metadata_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SetModificationTimeRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_metadata_proto_metadata_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetModificationTimeRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_metadata_proto_metadata_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetModificationTimeResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_metadata_proto_metadata_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetModificationTimesRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_metadata_proto_metadata_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetModificationTimesResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_metadata_proto_metadata_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RemoveMetadataRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_metadata_proto_metadata_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetFoldersRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_metadata_proto_metadata_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetFoldersResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_metadata_proto_metadata_proto_rawDesc, + NumEnums: 0, + NumMessages: 9, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_metadata_proto_metadata_proto_goTypes, + DependencyIndexes: file_metadata_proto_metadata_proto_depIdxs, + MessageInfos: file_metadata_proto_metadata_proto_msgTypes, + }.Build() + File_metadata_proto_metadata_proto = out.File + file_metadata_proto_metadata_proto_rawDesc = nil + file_metadata_proto_metadata_proto_goTypes = nil + file_metadata_proto_metadata_proto_depIdxs = nil +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConnInterface + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion6 + +// MetadataClient is the client API for Metadata service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. +type MetadataClient interface { + SetModificationTime(ctx context.Context, in *SetModificationTimeRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + GetModificationTime(ctx context.Context, in *GetModificationTimeRequest, opts ...grpc.CallOption) (*GetModificationTimeResponse, error) + GetModificationTimes(ctx context.Context, in *GetModificationTimesRequest, opts ...grpc.CallOption) (*GetModificationTimesResponse, error) + RemoveMetadata(ctx context.Context, in *RemoveMetadataRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + GetFolders(ctx context.Context, in *GetFoldersRequest, opts ...grpc.CallOption) (*GetFoldersResponse, error) +} + +type metadataClient struct { + cc grpc.ClientConnInterface +} + +func NewMetadataClient(cc grpc.ClientConnInterface) MetadataClient { + return &metadataClient{cc} +} + +func (c *metadataClient) SetModificationTime(ctx context.Context, in *SetModificationTimeRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, "/proto.Metadata/SetModificationTime", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *metadataClient) GetModificationTime(ctx context.Context, in *GetModificationTimeRequest, opts ...grpc.CallOption) (*GetModificationTimeResponse, error) { + out := new(GetModificationTimeResponse) + err := c.cc.Invoke(ctx, "/proto.Metadata/GetModificationTime", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *metadataClient) GetModificationTimes(ctx context.Context, in *GetModificationTimesRequest, opts ...grpc.CallOption) (*GetModificationTimesResponse, error) { + out := new(GetModificationTimesResponse) + err := c.cc.Invoke(ctx, "/proto.Metadata/GetModificationTimes", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *metadataClient) RemoveMetadata(ctx context.Context, in *RemoveMetadataRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, "/proto.Metadata/RemoveMetadata", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *metadataClient) GetFolders(ctx context.Context, in *GetFoldersRequest, opts ...grpc.CallOption) (*GetFoldersResponse, error) { + out := new(GetFoldersResponse) + err := c.cc.Invoke(ctx, "/proto.Metadata/GetFolders", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// MetadataServer is the server API for Metadata service. +type MetadataServer interface { + SetModificationTime(context.Context, *SetModificationTimeRequest) (*emptypb.Empty, error) + GetModificationTime(context.Context, *GetModificationTimeRequest) (*GetModificationTimeResponse, error) + GetModificationTimes(context.Context, *GetModificationTimesRequest) (*GetModificationTimesResponse, error) + RemoveMetadata(context.Context, *RemoveMetadataRequest) (*emptypb.Empty, error) + GetFolders(context.Context, *GetFoldersRequest) (*GetFoldersResponse, error) +} + +// UnimplementedMetadataServer can be embedded to have forward compatible implementations. +type UnimplementedMetadataServer struct { +} + +func (*UnimplementedMetadataServer) SetModificationTime(context.Context, *SetModificationTimeRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method SetModificationTime not implemented") +} +func (*UnimplementedMetadataServer) GetModificationTime(context.Context, *GetModificationTimeRequest) (*GetModificationTimeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetModificationTime not implemented") +} +func (*UnimplementedMetadataServer) GetModificationTimes(context.Context, *GetModificationTimesRequest) (*GetModificationTimesResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetModificationTimes not implemented") +} +func (*UnimplementedMetadataServer) RemoveMetadata(context.Context, *RemoveMetadataRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method RemoveMetadata not implemented") +} +func (*UnimplementedMetadataServer) GetFolders(context.Context, *GetFoldersRequest) (*GetFoldersResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetFolders not implemented") +} + +func RegisterMetadataServer(s *grpc.Server, srv MetadataServer) { + s.RegisterService(&_Metadata_serviceDesc, srv) +} + +func _Metadata_SetModificationTime_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SetModificationTimeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MetadataServer).SetModificationTime(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/proto.Metadata/SetModificationTime", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MetadataServer).SetModificationTime(ctx, req.(*SetModificationTimeRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Metadata_GetModificationTime_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetModificationTimeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MetadataServer).GetModificationTime(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/proto.Metadata/GetModificationTime", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MetadataServer).GetModificationTime(ctx, req.(*GetModificationTimeRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Metadata_GetModificationTimes_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetModificationTimesRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MetadataServer).GetModificationTimes(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/proto.Metadata/GetModificationTimes", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MetadataServer).GetModificationTimes(ctx, req.(*GetModificationTimesRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Metadata_RemoveMetadata_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RemoveMetadataRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MetadataServer).RemoveMetadata(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/proto.Metadata/RemoveMetadata", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MetadataServer).RemoveMetadata(ctx, req.(*RemoveMetadataRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Metadata_GetFolders_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetFoldersRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MetadataServer).GetFolders(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/proto.Metadata/GetFolders", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MetadataServer).GetFolders(ctx, req.(*GetFoldersRequest)) + } + return interceptor(ctx, in, info, handler) +} + +var _Metadata_serviceDesc = grpc.ServiceDesc{ + ServiceName: "proto.Metadata", + HandlerType: (*MetadataServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "SetModificationTime", + Handler: _Metadata_SetModificationTime_Handler, + }, + { + MethodName: "GetModificationTime", + Handler: _Metadata_GetModificationTime_Handler, + }, + { + MethodName: "GetModificationTimes", + Handler: _Metadata_GetModificationTimes_Handler, + }, + { + MethodName: "RemoveMetadata", + Handler: _Metadata_RemoveMetadata_Handler, + }, + { + MethodName: "GetFolders", + Handler: _Metadata_GetFolders_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "metadata/proto/metadata.proto", +} diff --git a/sdk/plugin/metadata/proto/metadata.proto b/sdk/plugin/metadata/proto/metadata.proto new file mode 100644 index 00000000..9c3cf8ef --- /dev/null +++ b/sdk/plugin/metadata/proto/metadata.proto @@ -0,0 +1,54 @@ +syntax = "proto3"; +package proto; + +import "google/protobuf/empty.proto"; + +option go_package = "sdk/plugin/metadata/proto"; + +message SetModificationTimeRequest { + string storage_id = 1; + string object_path = 2; + int64 modification_time = 3; +} + +message GetModificationTimeRequest { + string storage_id = 1; + string object_path = 2; +} + +message GetModificationTimeResponse { + int64 modification_time = 1; +} + +message GetModificationTimesRequest { + string storage_id = 1; + string folder_path = 2; +} + +message GetModificationTimesResponse { + // the file name (not the full path) is the map key and the modification time is the map value + map pairs = 1; +} + +message RemoveMetadataRequest { + string storage_id = 1; + string object_path = 2; +} + +message GetFoldersRequest { + string storage_id = 1; + int32 limit = 2; + string from = 3; +} + +message GetFoldersResponse { + repeated string folders = 1; +} + +service Metadata { + rpc SetModificationTime(SetModificationTimeRequest) returns (google.protobuf.Empty); + rpc GetModificationTime(GetModificationTimeRequest) returns (GetModificationTimeResponse); + rpc GetModificationTimes(GetModificationTimesRequest) returns (GetModificationTimesResponse); + rpc RemoveMetadata(RemoveMetadataRequest) returns (google.protobuf.Empty); + rpc GetFolders(GetFoldersRequest) returns (GetFoldersResponse); +} \ No newline at end of file diff --git a/sdk/plugin/mkproto.sh b/sdk/plugin/mkproto.sh index c138d82f..b1b775f1 100755 --- a/sdk/plugin/mkproto.sh +++ b/sdk/plugin/mkproto.sh @@ -4,3 +4,4 @@ protoc notifier/proto/notifier.proto --go_out=plugins=grpc:../.. --go_out=../../ protoc kms/proto/kms.proto --go_out=plugins=grpc:../.. --go_out=../../.. protoc auth/proto/auth.proto --go_out=plugins=grpc:../.. --go_out=../../.. protoc eventsearcher/proto/search.proto --go_out=plugins=grpc:../.. --go_out=../../.. +protoc metadata/proto/metadata.proto --go_out=plugins=grpc:../.. --go_out=../../.. diff --git a/sdk/plugin/plugin.go b/sdk/plugin/plugin.go index b62a0b8a..179894d2 100644 --- a/sdk/plugin/plugin.go +++ b/sdk/plugin/plugin.go @@ -16,6 +16,7 @@ import ( "github.com/drakkan/sftpgo/v2/sdk/plugin/auth" "github.com/drakkan/sftpgo/v2/sdk/plugin/eventsearcher" kmsplugin "github.com/drakkan/sftpgo/v2/sdk/plugin/kms" + "github.com/drakkan/sftpgo/v2/sdk/plugin/metadata" "github.com/drakkan/sftpgo/v2/sdk/plugin/notifier" "github.com/drakkan/sftpgo/v2/util" ) @@ -30,6 +31,8 @@ var ( pluginsLogLevel = hclog.Debug // ErrNoSearcher defines the error to return for events searches if no plugin is configured ErrNoSearcher = errors.New("no events searcher plugin defined") + // ErrNoMetadater returns the error to return for metadata methods if no plugin is configured + ErrNoMetadater = errors.New("no metadata plugin defined") ) // Renderer defines the interface for generic objects rendering @@ -78,17 +81,20 @@ type Manager struct { closed int32 done chan bool // List of configured plugins - Configs []Config `json:"plugins" mapstructure:"plugins"` - notifLock sync.RWMutex - notifiers []*notifierPlugin - kmsLock sync.RWMutex - kms []*kmsPlugin - authLock sync.RWMutex - auths []*authPlugin - searcherLock sync.RWMutex - searcher *searcherPlugin - authScopes int - hasSearcher bool + Configs []Config `json:"plugins" mapstructure:"plugins"` + notifLock sync.RWMutex + notifiers []*notifierPlugin + kmsLock sync.RWMutex + kms []*kmsPlugin + authLock sync.RWMutex + auths []*authPlugin + searcherLock sync.RWMutex + searcher *searcherPlugin + metadaterLock sync.RWMutex + metadater *metadataPlugin + authScopes int + hasSearcher bool + hasMetadater bool } // Initialize initializes the configured plugins @@ -100,19 +106,15 @@ func Initialize(configs []Config, logVerbose bool) error { closed: 0, authScopes: -1, } + setLogLevel(logVerbose) if len(configs) == 0 { return nil } + if err := Handler.validateConfigs(); err != nil { return err } - if logVerbose { - pluginsLogLevel = hclog.Debug - } else { - pluginsLogLevel = hclog.Info - } - kmsID := 0 for idx, config := range Handler.Configs { switch config.Type { @@ -151,6 +153,12 @@ func Initialize(configs []Config, logVerbose bool) error { return err } Handler.searcher = plugin + case metadata.PluginName: + plugin, err := newMetadaterPlugin(config) + if err != nil { + return err + } + Handler.metadater = plugin default: return fmt.Errorf("unsupported plugin type: %v", config.Type) } @@ -163,6 +171,7 @@ func (m *Manager) validateConfigs() error { kmsSchemes := make(map[string]bool) kmsEncryptions := make(map[string]bool) m.hasSearcher = false + m.hasMetadater = false for _, config := range m.Configs { if config.Type == kmsplugin.PluginName { @@ -177,10 +186,16 @@ func (m *Manager) validateConfigs() error { } if config.Type == eventsearcher.PluginName { if m.hasSearcher { - return fmt.Errorf("only one eventsearcher plugin can be defined") + return errors.New("only one eventsearcher plugin can be defined") } m.hasSearcher = true } + if config.Type == metadata.PluginName { + if m.hasMetadater { + return errors.New("only one metadata plugin can be defined") + } + m.hasMetadater = true + } } return nil } @@ -242,6 +257,71 @@ func (m *Manager) SearchProviderEvents(startTimestamp, endTimestamp int64, usern order, actions, objectTypes, instanceIDs, excludeIDs) } +// HasMetadater returns true if a metadata plugin is defined +func (m *Manager) HasMetadater() bool { + return m.hasMetadater +} + +// SetModificationTime sets the modification time for the specified object +func (m *Manager) SetModificationTime(storageID, objectPath string, mTime int64) error { + if !m.hasMetadater { + return ErrNoMetadater + } + m.metadaterLock.RLock() + plugin := m.metadater + m.metadaterLock.RUnlock() + + return plugin.metadater.SetModificationTime(storageID, objectPath, mTime) +} + +// GetModificationTime returns the modification time for the specified path +func (m *Manager) GetModificationTime(storageID, objectPath string, isDir bool) (int64, error) { + if !m.hasMetadater { + return 0, ErrNoMetadater + } + m.metadaterLock.RLock() + plugin := m.metadater + m.metadaterLock.RUnlock() + + return plugin.metadater.GetModificationTime(storageID, objectPath) +} + +// GetModificationTimes returns the modification times for all the files within the specified folder +func (m *Manager) GetModificationTimes(storageID, objectPath string) (map[string]int64, error) { + if !m.hasMetadater { + return nil, ErrNoMetadater + } + m.metadaterLock.RLock() + plugin := m.metadater + m.metadaterLock.RUnlock() + + return plugin.metadater.GetModificationTimes(storageID, objectPath) +} + +// RemoveMetadata deletes the metadata stored for the specified object +func (m *Manager) RemoveMetadata(storageID, objectPath string) error { + if !m.hasMetadater { + return ErrNoMetadater + } + m.metadaterLock.RLock() + plugin := m.metadater + m.metadaterLock.RUnlock() + + return plugin.metadater.RemoveMetadata(storageID, objectPath) +} + +// GetMetadataFolders returns the folders that metadata is associated with +func (m *Manager) GetMetadataFolders(storageID, from string, limit int) ([]string, error) { + if !m.hasMetadater { + return nil, ErrNoMetadater + } + m.metadaterLock.RLock() + plugin := m.metadater + m.metadaterLock.RUnlock() + + return plugin.metadater.GetFolders(storageID, limit, from) +} + func (m *Manager) kmsEncrypt(secret kms.BaseSecret, url string, masterKey string, kmsID int) (string, string, int32, error) { m.kmsLock.RLock() plugin := m.kms[kmsID] @@ -417,6 +497,7 @@ func (m *Manager) checkCrashedPlugins() { } } m.authLock.RUnlock() + if m.hasSearcher { m.searcherLock.RLock() if m.searcher.exited() { @@ -426,6 +507,16 @@ func (m *Manager) checkCrashedPlugins() { } m.searcherLock.RUnlock() } + + if m.hasMetadater { + m.metadaterLock.RLock() + if m.metadater.exited() { + defer func(cfg Config) { + Handler.restartMetadaterPlugin(cfg) + }(m.metadater.config) + } + m.metadaterLock.RUnlock() + } } func (m *Manager) restartNotifierPlugin(config Config, idx int) { @@ -494,6 +585,22 @@ func (m *Manager) restartSearcherPlugin(config Config) { m.searcherLock.Unlock() } +func (m *Manager) restartMetadaterPlugin(config Config) { + if atomic.LoadInt32(&m.closed) == 1 { + return + } + logger.Info(logSender, "", "try to restart crashed metadater plugin %#v", config.Cmd) + plugin, err := newMetadaterPlugin(config) + if err != nil { + logger.Warn(logSender, "", "unable to restart metadater plugin %#v, err: %v", config.Cmd, err) + return + } + + m.metadaterLock.Lock() + m.metadater = plugin + m.metadaterLock.Unlock() +} + // Cleanup releases all the active plugins func (m *Manager) Cleanup() { logger.Debug(logSender, "", "cleanup") @@ -526,6 +633,21 @@ func (m *Manager) Cleanup() { m.searcher.cleanup() m.searcherLock.Unlock() } + + if m.hasMetadater { + m.metadaterLock.Lock() + logger.Debug(logSender, "", "cleanup metadater plugin %v", m.metadater.config.Cmd) + m.metadater.cleanup() + m.metadaterLock.Unlock() + } +} + +func setLogLevel(logVerbose bool) { + if logVerbose { + pluginsLogLevel = hclog.Debug + } else { + pluginsLogLevel = hclog.Info + } } func startCheckTicker() { diff --git a/sftpd/handler.go b/sftpd/handler.go index e5aff3c9..5c0d10dd 100644 --- a/sftpd/handler.go +++ b/sftpd/handler.go @@ -331,7 +331,7 @@ func (c *Connection) handleSFTPRemove(request *sftp.Request) error { var fi os.FileInfo if fi, err = fs.Lstat(fsPath); err != nil { - c.Log(logger.LevelDebug, "failed to remove a file %#v: stat error: %+v", fsPath, err) + c.Log(logger.LevelDebug, "failed to remove file %#v: stat error: %+v", fsPath, err) return c.GetFsError(fs, err) } if fi.IsDir() && fi.Mode()&os.ModeSymlink == 0 { diff --git a/vfs/azblobfs.go b/vfs/azblobfs.go index 3668276c..81f5411f 100644 --- a/vfs/azblobfs.go +++ b/vfs/azblobfs.go @@ -26,6 +26,8 @@ import ( "github.com/drakkan/sftpgo/v2/logger" "github.com/drakkan/sftpgo/v2/metric" + "github.com/drakkan/sftpgo/v2/sdk/plugin" + "github.com/drakkan/sftpgo/v2/util" "github.com/drakkan/sftpgo/v2/version" ) @@ -104,6 +106,7 @@ func NewAzBlobFs(connectionID, localTempDir, mountPath string, config AzBlobFsCo return fs, fmt.Errorf("container name in SAS URL %#v and container provided %#v do not match", parts.ContainerName, fs.config.Container) } + fs.config.Container = parts.ContainerName fs.svc = nil fs.containerURL = azblob.NewContainerURL(*u, pipeline) } else { @@ -168,17 +171,18 @@ func (fs *AzureBlobFs) Stat(name string) (os.FileInfo, error) { return nil, err } } - return NewFileInfo(name, true, 0, time.Now(), false), nil + return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, true, 0, time.Now(), false)) } if fs.config.KeyPrefix == name+"/" { - return NewFileInfo(name, true, 0, time.Now(), false), nil + return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, true, 0, time.Now(), false)) } attrs, err := fs.headObject(name) if err == nil { isDir := (attrs.ContentType() == dirMimeType) metric.AZListObjectsCompleted(nil) - return NewFileInfo(name, isDir, attrs.ContentLength(), attrs.LastModified(), false), nil + return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, isDir, attrs.ContentLength(), + attrs.LastModified(), false)) } if !fs.IsNotExist(err) { return nil, err @@ -189,7 +193,7 @@ func (fs *AzureBlobFs) Stat(name string) (os.FileInfo, error) { return nil, err } if hasContents { - return NewFileInfo(name, true, 0, time.Now(), false), nil + return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, true, 0, time.Now(), false)) } return nil, errors.New("404 no such file or directory") } @@ -335,6 +339,16 @@ func (fs *AzureBlobFs) Rename(source, target string) error { return err } metric.AZCopyObjectCompleted(nil) + if plugin.Handler.HasMetadater() { + if !fi.IsDir() { + err = plugin.Handler.SetModificationTime(fs.getStorageID(), ensureAbsPath(target), + util.GetTimeAsMsSinceEpoch(fi.ModTime())) + if err != nil { + fsLog(fs, logger.LevelWarn, "unable to preserve modification time after renaming %#v -> %#v: %v", + source, target, err) + } + } + } return fs.Remove(source, fi.IsDir()) } @@ -355,6 +369,11 @@ func (fs *AzureBlobFs) Remove(name string, isDir bool) error { _, err := blobBlockURL.Delete(ctx, azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}) metric.AZDeleteObjectCompleted(err) + if plugin.Handler.HasMetadater() && err == nil && !isDir { + if errMetadata := plugin.Handler.RemoveMetadata(fs.getStorageID(), ensureAbsPath(name)); errMetadata != nil { + fsLog(fs, logger.LevelWarn, "unable to remove metadata for path %#v: %v", name, errMetadata) + } + } return err } @@ -397,8 +416,22 @@ func (*AzureBlobFs) Chmod(name string, mode os.FileMode) error { } // Chtimes changes the access and modification times of the named file. -func (*AzureBlobFs) Chtimes(name string, atime, mtime time.Time) error { - return ErrVfsUnsupported +func (fs *AzureBlobFs) Chtimes(name string, atime, mtime time.Time, isUploading bool) error { + if !plugin.Handler.HasMetadater() { + return ErrVfsUnsupported + } + if !isUploading { + info, err := fs.Stat(name) + if err != nil { + return err + } + if info.IsDir() { + return ErrVfsUnsupported + } + } + + return plugin.Handler.SetModificationTime(fs.getStorageID(), ensureAbsPath(name), + util.GetTimeAsMsSinceEpoch(mtime)) } // Truncate changes the size of the named file. @@ -413,14 +446,12 @@ func (*AzureBlobFs) Truncate(name string, size int64) error { func (fs *AzureBlobFs) ReadDir(dirname string) ([]os.FileInfo, error) { var result []os.FileInfo // dirname must be already cleaned - prefix := "" - if dirname != "" && dirname != "." { - prefix = strings.TrimPrefix(dirname, "/") - if !strings.HasSuffix(prefix, "/") { - prefix += "/" - } - } + prefix := fs.getPrefix(dirname) + modTimes, err := getFolderModTimes(fs.getStorageID(), dirname) + if err != nil { + return result, err + } prefixes := make(map[string]bool) for marker := (azblob.Marker{}); marker.NotDone(); { @@ -473,7 +504,11 @@ func (fs *AzureBlobFs) ReadDir(dirname string) ([]os.FileInfo, error) { prefixes[name] = true } } - result = append(result, NewFileInfo(name, isDir, size, blobInfo.Properties.LastModified, false)) + modTime := blobInfo.Properties.LastModified + if t, ok := modTimes[name]; ok { + modTime = util.GetTimeFromMsecSinceEpoch(t) + } + result = append(result, NewFileInfo(name, isDir, size, modTime, false)) } } @@ -596,6 +631,54 @@ func (fs *AzureBlobFs) ScanRootDirContents() (int, int64, error) { return numFiles, size, nil } +func (fs *AzureBlobFs) getFileNamesInPrefix(fsPrefix string) (map[string]bool, error) { + fileNames := make(map[string]bool) + prefix := "" + if fsPrefix != "/" { + prefix = strings.TrimPrefix(fsPrefix, "/") + } + + for marker := (azblob.Marker{}); marker.NotDone(); { + ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout)) + defer cancelFn() + + listBlob, err := fs.containerURL.ListBlobsHierarchySegment(ctx, marker, "/", azblob.ListBlobsSegmentOptions{ + Details: azblob.BlobListingDetails{ + Copy: false, + Metadata: false, + Snapshots: false, + UncommittedBlobs: false, + Deleted: false, + }, + Prefix: prefix, + }) + if err != nil { + metric.AZListObjectsCompleted(err) + return fileNames, err + } + marker = listBlob.NextMarker + for idx := range listBlob.Segment.BlobItems { + blobInfo := &listBlob.Segment.BlobItems[idx] + name := strings.TrimPrefix(blobInfo.Name, prefix) + if blobInfo.Properties.ContentType != nil { + if *blobInfo.Properties.ContentType == dirMimeType { + continue + } + } + + fileNames[name] = true + } + } + + metric.AZListObjectsCompleted(nil) + return fileNames, nil +} + +// CheckMetadata checks the metadata consistency +func (fs *AzureBlobFs) CheckMetadata() error { + return fsMetadataCheck(fs, fs.getStorageID(), fs.config.KeyPrefix) +} + // GetDirSize returns the number of files and the size for a folder // including any subfolders func (*AzureBlobFs) GetDirSize(dirname string) (int, int64, error) { @@ -733,6 +816,17 @@ func (*AzureBlobFs) GetAvailableDiskSize(dirName string) (*sftp.StatVFS, error) return nil, ErrStorageSizeUnavailable } +func (*AzureBlobFs) getPrefix(name string) string { + prefix := "" + if name != "" && name != "." { + prefix = strings.TrimPrefix(name, "/") + if !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + } + return prefix +} + func (fs *AzureBlobFs) isEqual(key string, virtualName string) bool { if key == virtualName { return true @@ -905,6 +999,16 @@ func (fs *AzureBlobFs) incrementBlockID(blockID []byte) { } } +func (fs *AzureBlobFs) getStorageID() string { + if fs.config.Endpoint != "" { + if !strings.HasSuffix(fs.config.Endpoint, "/") { + return fmt.Sprintf("azblob://%v/%v", fs.config.Endpoint, fs.config.Container) + } + return fmt.Sprintf("azblob://%v%v", fs.config.Endpoint, fs.config.Container) + } + return fmt.Sprintf("azblob://%v", fs.config.Container) +} + type bufferAllocator struct { sync.Mutex available [][]byte diff --git a/vfs/folder.go b/vfs/folder.go index b62cf088..c84aa375 100644 --- a/vfs/folder.go +++ b/vfs/folder.go @@ -192,6 +192,18 @@ func (v *VirtualFolder) GetFilesystem(connectionID string, forbiddenSelfUsers [] } } +// CheckMetadataConsistency checks the consistency between the metadata stored +// in the configured metadata plugin and the filesystem +func (v *VirtualFolder) CheckMetadataConsistency() error { + fs, err := v.GetFilesystem("", nil) + if err != nil { + return err + } + defer fs.Close() + + return fs.CheckMetadata() +} + // ScanQuota scans the folder and returns the number of files and their size func (v *VirtualFolder) ScanQuota() (int, int64, error) { fs, err := v.GetFilesystem("", nil) diff --git a/vfs/gcsfs.go b/vfs/gcsfs.go index 9f7d02b5..5ce06ff6 100644 --- a/vfs/gcsfs.go +++ b/vfs/gcsfs.go @@ -26,6 +26,8 @@ import ( "github.com/drakkan/sftpgo/v2/kms" "github.com/drakkan/sftpgo/v2/logger" "github.com/drakkan/sftpgo/v2/metric" + "github.com/drakkan/sftpgo/v2/sdk/plugin" + "github.com/drakkan/sftpgo/v2/util" "github.com/drakkan/sftpgo/v2/version" ) @@ -117,10 +119,10 @@ func (fs *GCSFs) Stat(name string) (os.FileInfo, error) { if err != nil { return nil, err } - return NewFileInfo(name, true, 0, time.Now(), false), nil + return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, true, 0, time.Now(), false)) } if fs.config.KeyPrefix == name+"/" { - return NewFileInfo(name, true, 0, time.Now(), false), nil + return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, true, 0, time.Now(), false)) } _, info, err := fs.getObjectStat(name) return info, err @@ -255,6 +257,16 @@ func (fs *GCSFs) Rename(source, target string) error { if err != nil { return err } + if plugin.Handler.HasMetadater() { + if !fi.IsDir() { + err = plugin.Handler.SetModificationTime(fs.getStorageID(), ensureAbsPath(target), + util.GetTimeAsMsSinceEpoch(fi.ModTime())) + if err != nil { + fsLog(fs, logger.LevelWarn, "unable to preserve modification time after renaming %#v -> %#v: %v", + source, target, err) + } + } + } return fs.Remove(source, fi.IsDir()) } @@ -281,6 +293,11 @@ func (fs *GCSFs) Remove(name string, isDir bool) error { err = fs.svc.Bucket(fs.config.Bucket).Object(strings.TrimSuffix(name, "/")).Delete(ctx) } metric.GCSDeleteObjectCompleted(err) + if plugin.Handler.HasMetadater() && err == nil && !isDir { + if errMetadata := plugin.Handler.RemoveMetadata(fs.getStorageID(), ensureAbsPath(name)); errMetadata != nil { + fsLog(fs, logger.LevelWarn, "unable to remove metadata for path %#v: %v", name, errMetadata) + } + } return err } @@ -326,8 +343,22 @@ func (*GCSFs) Chmod(name string, mode os.FileMode) error { } // Chtimes changes the access and modification times of the named file. -func (*GCSFs) Chtimes(name string, atime, mtime time.Time) error { - return ErrVfsUnsupported +func (fs *GCSFs) Chtimes(name string, atime, mtime time.Time, isUploading bool) error { + if !plugin.Handler.HasMetadater() { + return ErrVfsUnsupported + } + if !isUploading { + info, err := fs.Stat(name) + if err != nil { + return err + } + if info.IsDir() { + return ErrVfsUnsupported + } + } + + return plugin.Handler.SetModificationTime(fs.getStorageID(), ensureAbsPath(name), + util.GetTimeAsMsSinceEpoch(mtime)) } // Truncate changes the size of the named file. @@ -350,6 +381,11 @@ func (fs *GCSFs) ReadDir(dirname string) ([]os.FileInfo, error) { return nil, err } + modTimes, err := getFolderModTimes(fs.getStorageID(), dirname) + if err != nil { + return result, err + } + prefixes := make(map[string]bool) ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout)) defer cancelFn() @@ -393,8 +429,11 @@ func (fs *GCSFs) ReadDir(dirname string) ([]os.FileInfo, error) { } prefixes[name] = true } - fi := NewFileInfo(name, isDir, attrs.Size, attrs.Updated, false) - result = append(result, fi) + modTime := attrs.Updated + if t, ok := modTimes[name]; ok { + modTime = util.GetTimeFromMsecSinceEpoch(t) + } + result = append(result, NewFileInfo(name, isDir, attrs.Size, modTime, false)) } } metric.GCSListObjectsCompleted(nil) @@ -497,6 +536,58 @@ func (fs *GCSFs) ScanRootDirContents() (int, int64, error) { return numFiles, size, err } +func (fs *GCSFs) getFileNamesInPrefix(fsPrefix string) (map[string]bool, error) { + fileNames := make(map[string]bool) + prefix := "" + if fsPrefix != "/" { + prefix = strings.TrimPrefix(fsPrefix, "/") + } + + query := &storage.Query{ + Prefix: prefix, + Delimiter: "/", + } + err := query.SetAttrSelection(gcsDefaultFieldsSelection) + if err != nil { + return fileNames, err + } + ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout)) + defer cancelFn() + + bkt := fs.svc.Bucket(fs.config.Bucket) + it := bkt.Objects(ctx, query) + for { + attrs, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + metric.GCSListObjectsCompleted(err) + return fileNames, err + } + if !attrs.Deleted.IsZero() { + continue + } + if attrs.Prefix == "" { + name, isDir := fs.resolve(attrs.Name, prefix) + if name == "" { + continue + } + if isDir || attrs.ContentType == dirMimeType { + continue + } + fileNames[name] = true + } + } + metric.GCSListObjectsCompleted(nil) + return fileNames, nil +} + +// CheckMetadata checks the metadata consistency +func (fs *GCSFs) CheckMetadata() error { + return fsMetadataCheck(fs, fs.getStorageID(), fs.config.KeyPrefix) +} + // GetDirSize returns the number of files and the size for a folder // including any subfolders func (*GCSFs) GetDirSize(dirname string) (int, int64, error) { @@ -617,11 +708,13 @@ func (fs *GCSFs) resolve(name string, prefix string) (string, bool) { // getObjectStat returns the stat result and the real object name as first value func (fs *GCSFs) getObjectStat(name string) (string, os.FileInfo, error) { attrs, err := fs.headObject(name) + var info os.FileInfo if err == nil { objSize := attrs.Size objectModTime := attrs.Updated isDir := attrs.ContentType == dirMimeType || strings.HasSuffix(attrs.Name, "/") - return name, NewFileInfo(name, isDir, objSize, objectModTime, false), nil + info, err = updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, isDir, objSize, objectModTime, false)) + return name, info, err } if !fs.IsNotExist(err) { return "", nil, err @@ -632,16 +725,16 @@ func (fs *GCSFs) getObjectStat(name string) (string, os.FileInfo, error) { return "", nil, err } if hasContents { - return name, NewFileInfo(name, true, 0, time.Now(), false), nil + info, err = updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, true, 0, time.Now(), false)) + return name, info, err } // finally check if this is an object with a trailing / attrs, err = fs.headObject(name + "/") if err != nil { return "", nil, err } - objSize := attrs.Size - objectModTime := attrs.Updated - return name + "/", NewFileInfo(name, true, objSize, objectModTime, false), nil + info, err = updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, true, attrs.Size, attrs.Updated, false)) + return name + "/", info, err } func (fs *GCSFs) checkIfBucketExists() error { @@ -735,3 +828,7 @@ func (fs *GCSFs) Close() error { func (*GCSFs) GetAvailableDiskSize(dirName string) (*sftp.StatVFS, error) { return nil, ErrStorageSizeUnavailable } + +func (fs *GCSFs) getStorageID() string { + return fmt.Sprintf("gs://%v", fs.config.Bucket) +} diff --git a/vfs/osfs.go b/vfs/osfs.go index 2bfeee9e..37fd6648 100644 --- a/vfs/osfs.go +++ b/vfs/osfs.go @@ -161,7 +161,7 @@ func (*OsFs) Chmod(name string, mode os.FileMode) error { } // Chtimes changes the access and modification times of the named file -func (*OsFs) Chtimes(name string, atime, mtime time.Time) error { +func (*OsFs) Chtimes(name string, atime, mtime time.Time, isUploading bool) error { return os.Chtimes(name, atime, mtime) } @@ -239,6 +239,11 @@ func (fs *OsFs) ScanRootDirContents() (int, int64, error) { return fs.GetDirSize(fs.rootDir) } +// CheckMetadata checks the metadata consistency +func (*OsFs) CheckMetadata() error { + return nil +} + // GetAtomicUploadPath returns the path to use for an atomic upload func (*OsFs) GetAtomicUploadPath(name string) string { dir := filepath.Dir(name) diff --git a/vfs/s3fs.go b/vfs/s3fs.go index 50006386..264d7602 100644 --- a/vfs/s3fs.go +++ b/vfs/s3fs.go @@ -26,6 +26,7 @@ import ( "github.com/drakkan/sftpgo/v2/logger" "github.com/drakkan/sftpgo/v2/metric" + "github.com/drakkan/sftpgo/v2/sdk/plugin" "github.com/drakkan/sftpgo/v2/util" "github.com/drakkan/sftpgo/v2/version" ) @@ -136,7 +137,7 @@ func (fs *S3Fs) Stat(name string) (os.FileInfo, error) { if err != nil { return result, err } - return NewFileInfo(name, true, 0, time.Now(), false), nil + return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, true, 0, time.Now(), false)) } if "/"+fs.config.KeyPrefix == name+"/" { return NewFileInfo(name, true, 0, time.Now(), false), nil @@ -146,7 +147,7 @@ func (fs *S3Fs) Stat(name string) (os.FileInfo, error) { // a "dir" has a trailing "/" so we cannot have a directory here objSize := *obj.ContentLength objectModTime := *obj.LastModified - return NewFileInfo(name, false, objSize, objectModTime, false), nil + return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, false, objSize, objectModTime, false)) } if !fs.IsNotExist(err) { return result, err @@ -154,7 +155,7 @@ func (fs *S3Fs) Stat(name string) (os.FileInfo, error) { // now check if this is a prefix (virtual directory) hasContents, err := fs.hasContents(name) if err == nil && hasContents { - return NewFileInfo(name, true, 0, time.Now(), false), nil + return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, true, 0, time.Now(), false)) } else if err != nil { return nil, err } @@ -173,7 +174,7 @@ func (fs *S3Fs) getStatForDir(name string) (os.FileInfo, error) { } objSize := *obj.ContentLength objectModTime := *obj.LastModified - return NewFileInfo(name, true, objSize, objectModTime, false), nil + return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, true, objSize, objectModTime, false)) } // Lstat returns a FileInfo describing the named file @@ -322,6 +323,16 @@ func (fs *S3Fs) Rename(source, target string) error { if err != nil { return err } + if plugin.Handler.HasMetadater() { + if !fi.IsDir() { + err = plugin.Handler.SetModificationTime(fs.getStorageID(), ensureAbsPath(target), + util.GetTimeAsMsSinceEpoch(fi.ModTime())) + if err != nil { + fsLog(fs, logger.LevelWarn, "unable to preserve modification time after renaming %#v -> %#v: %v", + source, target, err) + } + } + } return fs.Remove(source, fi.IsDir()) } @@ -346,6 +357,11 @@ func (fs *S3Fs) Remove(name string, isDir bool) error { Key: aws.String(name), }) metric.S3DeleteObjectCompleted(err) + if plugin.Handler.HasMetadater() && err == nil && !isDir { + if errMetadata := plugin.Handler.RemoveMetadata(fs.getStorageID(), ensureAbsPath(name)); errMetadata != nil { + fsLog(fs, logger.LevelWarn, "unable to remove metadata for path %#v: %v", name, errMetadata) + } + } return err } @@ -391,8 +407,21 @@ func (*S3Fs) Chmod(name string, mode os.FileMode) error { } // Chtimes changes the access and modification times of the named file. -func (*S3Fs) Chtimes(name string, atime, mtime time.Time) error { - return ErrVfsUnsupported +func (fs *S3Fs) Chtimes(name string, atime, mtime time.Time, isUploading bool) error { + if !plugin.Handler.HasMetadater() { + return ErrVfsUnsupported + } + if !isUploading { + info, err := fs.Stat(name) + if err != nil { + return err + } + if info.IsDir() { + return ErrVfsUnsupported + } + } + return plugin.Handler.SetModificationTime(fs.getStorageID(), ensureAbsPath(name), + util.GetTimeAsMsSinceEpoch(mtime)) } // Truncate changes the size of the named file. @@ -415,11 +444,15 @@ func (fs *S3Fs) ReadDir(dirname string) ([]os.FileInfo, error) { } } + modTimes, err := getFolderModTimes(fs.getStorageID(), dirname) + if err != nil { + return result, err + } prefixes := make(map[string]bool) ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout)) defer cancelFn() - err := fs.svc.ListObjectsV2PagesWithContext(ctx, &s3.ListObjectsV2Input{ + err = fs.svc.ListObjectsV2PagesWithContext(ctx, &s3.ListObjectsV2Input{ Bucket: aws.String(fs.config.Bucket), Prefix: aws.String(prefix), Delimiter: aws.String("/"), @@ -449,6 +482,9 @@ func (fs *S3Fs) ReadDir(dirname string) ([]os.FileInfo, error) { } prefixes[name] = true } + if t, ok := modTimes[name]; ok { + objectModTime = util.GetTimeFromMsecSinceEpoch(t) + } result = append(result, NewFileInfo(name, (isDir && objectSize == 0), objectSize, objectModTime, false)) } return true @@ -544,6 +580,41 @@ func (fs *S3Fs) ScanRootDirContents() (int, int64, error) { return numFiles, size, err } +func (fs *S3Fs) getFileNamesInPrefix(fsPrefix string) (map[string]bool, error) { + fileNames := make(map[string]bool) + prefix := "" + if fsPrefix != "/" { + prefix = strings.TrimPrefix(fsPrefix, "/") + } + ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout)) + defer cancelFn() + + err := fs.svc.ListObjectsV2PagesWithContext(ctx, &s3.ListObjectsV2Input{ + Bucket: aws.String(fs.config.Bucket), + Prefix: aws.String(prefix), + Delimiter: aws.String("/"), + }, func(page *s3.ListObjectsV2Output, lastPage bool) bool { + for _, fileObject := range page.Contents { + name, isDir := fs.resolve(fileObject.Key, prefix) + if name != "" && !isDir { + fileNames[name] = true + } + } + return true + }) + metric.S3ListObjectsCompleted(err) + if err != nil { + fsLog(fs, logger.LevelWarn, "unable to get content for prefix %#v: %v", prefix, err) + return nil, err + } + return fileNames, err +} + +// CheckMetadata checks the metadata consistency +func (fs *S3Fs) CheckMetadata() error { + return fsMetadataCheck(fs, fs.getStorageID(), fs.config.KeyPrefix) +} + // GetDirSize returns the number of files and the size for a folder // including any subfolders func (*S3Fs) GetDirSize(dirname string) (int, int64, error) { @@ -722,6 +793,16 @@ func (*S3Fs) GetAvailableDiskSize(dirName string) (*sftp.StatVFS, error) { return nil, ErrStorageSizeUnavailable } +func (fs *S3Fs) getStorageID() string { + if fs.config.Endpoint != "" { + if !strings.HasSuffix(fs.config.Endpoint, "/") { + return fmt.Sprintf("s3://%v/%v", fs.config.Endpoint, fs.config.Bucket) + } + return fmt.Sprintf("s3://%v%v", fs.config.Endpoint, fs.config.Bucket) + } + return fmt.Sprintf("s3://%v", fs.config.Bucket) +} + // ideally we should simply use url.PathEscape: // // https://github.com/awsdocs/aws-doc-sdk-examples/blob/master/go/example_code/s3/s3_copy_object.go#L65 diff --git a/vfs/sftpfs.go b/vfs/sftpfs.go index 01137180..5f386809 100644 --- a/vfs/sftpfs.go +++ b/vfs/sftpfs.go @@ -384,7 +384,7 @@ func (fs *SFTPFs) Chmod(name string, mode os.FileMode) error { } // Chtimes changes the access and modification times of the named file. -func (fs *SFTPFs) Chtimes(name string, atime, mtime time.Time) error { +func (fs *SFTPFs) Chtimes(name string, atime, mtime time.Time, isUploading bool) error { if err := fs.checkConnection(); err != nil { return err } @@ -464,6 +464,11 @@ func (fs *SFTPFs) ScanRootDirContents() (int, int64, error) { return fs.GetDirSize(fs.config.Prefix) } +// CheckMetadata checks the metadata consistency +func (*SFTPFs) CheckMetadata() error { + return nil +} + // GetAtomicUploadPath returns the path to use for an atomic upload func (*SFTPFs) GetAtomicUploadPath(name string) string { dir := path.Dir(name) diff --git a/vfs/vfs.go b/vfs/vfs.go index a15982d5..fa177e0c 100644 --- a/vfs/vfs.go +++ b/vfs/vfs.go @@ -19,6 +19,8 @@ import ( "github.com/drakkan/sftpgo/v2/kms" "github.com/drakkan/sftpgo/v2/logger" "github.com/drakkan/sftpgo/v2/sdk" + "github.com/drakkan/sftpgo/v2/sdk/plugin" + "github.com/drakkan/sftpgo/v2/sdk/plugin/metadata" "github.com/drakkan/sftpgo/v2/util" ) @@ -75,7 +77,7 @@ type Fs interface { Symlink(source, target string) error Chown(name string, uid int, gid int) error Chmod(name string, mode os.FileMode) error - Chtimes(name string, atime, mtime time.Time) error + Chtimes(name string, atime, mtime time.Time, isUploading bool) error Truncate(name string, size int64) error ReadDir(dirname string) ([]os.FileInfo, error) Readlink(name string) (string, error) @@ -95,9 +97,17 @@ type Fs interface { HasVirtualFolders() bool GetMimeType(name string) (string, error) GetAvailableDiskSize(dirName string) (*sftp.StatVFS, error) + CheckMetadata() error Close() error } +// fsMetadataChecker is a Fs that implements the getFileNamesInPrefix method. +// This interface is used to abstract metadata consistency checks +type fsMetadataChecker interface { + Fs + getFileNamesInPrefix(fsPrefix string) (map[string]bool, error) +} + // File defines an interface representing a SFTPGo file type File interface { io.Reader @@ -645,6 +655,102 @@ func SetPathPermissions(fs Fs, path string, uid int, gid int) { } } +func updateFileInfoModTime(storageID, objectPath string, info *FileInfo) (*FileInfo, error) { + if !plugin.Handler.HasMetadater() { + return info, nil + } + if info.IsDir() { + return info, nil + } + mTime, err := plugin.Handler.GetModificationTime(storageID, ensureAbsPath(objectPath), info.IsDir()) + if errors.Is(err, metadata.ErrNoSuchObject) { + return info, nil + } + if err != nil { + return info, err + } + info.modTime = util.GetTimeFromMsecSinceEpoch(mTime) + return info, nil +} + +func getFolderModTimes(storageID, dirName string) (map[string]int64, error) { + var err error + modTimes := make(map[string]int64) + if plugin.Handler.HasMetadater() { + modTimes, err = plugin.Handler.GetModificationTimes(storageID, ensureAbsPath(dirName)) + if err != nil && !errors.Is(err, metadata.ErrNoSuchObject) { + return modTimes, err + } + } + return modTimes, nil +} + +func ensureAbsPath(name string) string { + if path.IsAbs(name) { + return name + } + return path.Join("/", name) +} + +func fsMetadataCheck(fs fsMetadataChecker, storageID, keyPrefix string) error { + if !plugin.Handler.HasMetadater() { + return nil + } + limit := 100 + from := "" + for { + metadataFolders, err := plugin.Handler.GetMetadataFolders(storageID, from, limit) + if err != nil { + fsLog(fs, logger.LevelError, "unable to get folders: %v", err) + return err + } + for _, folder := range metadataFolders { + from = folder + fsPrefix := folder + if !strings.HasSuffix(folder, "/") { + fsPrefix += "/" + } + if keyPrefix != "" { + if !strings.HasPrefix(fsPrefix, "/"+keyPrefix) { + fsLog(fs, logger.LevelDebug, "skip metadata check for folder %#v outside prefix %#v", + folder, keyPrefix) + continue + } + } + fsLog(fs, logger.LevelDebug, "check metadata for folder %#v", folder) + metadataValues, err := plugin.Handler.GetModificationTimes(storageID, folder) + if err != nil { + fsLog(fs, logger.LevelError, "unable to get modification times for folder %#v: %v", folder, err) + return err + } + if len(metadataValues) == 0 { + fsLog(fs, logger.LevelDebug, "no metadata for folder %#v", folder) + continue + } + fileNames, err := fs.getFileNamesInPrefix(fsPrefix) + if err != nil { + fsLog(fs, logger.LevelError, "unable to get content for prefix %#v: %v", fsPrefix, err) + return err + } + // now check if we have metadata for a missing object + for k := range metadataValues { + if _, ok := fileNames[k]; !ok { + filePath := ensureAbsPath(path.Join(folder, k)) + if err = plugin.Handler.RemoveMetadata(storageID, filePath); err != nil { + fsLog(fs, logger.LevelError, "unable to remove metadata for missing file %#v: %v", filePath, err) + } else { + fsLog(fs, logger.LevelDebug, "metadata removed for missing file %#v", filePath) + } + } + } + } + + if len(metadataFolders) < limit { + return nil + } + } +} + func fsLog(fs Fs, level logger.LogLevel, format string, v ...interface{}) { logger.Log(level, fs.Name(), fs.ConnectionID(), format, v...) } diff --git a/webdavd/handler.go b/webdavd/handler.go index 4aa728f2..68ee50c4 100644 --- a/webdavd/handler.go +++ b/webdavd/handler.go @@ -106,7 +106,7 @@ func (c *Connection) RemoveAll(ctx context.Context, name string) error { var fi os.FileInfo if fi, err = fs.Lstat(p); err != nil { - c.Log(logger.LevelDebug, "failed to remove a file %#v: stat error: %+v", p, err) + c.Log(logger.LevelDebug, "failed to remove file %#v: stat error: %+v", p, err) return c.GetFsError(fs, err) }