beenull/sftpgo-mirror
1
0
Fork 0
mirror of https://github.com/drakkan/sftpgo.git synced 2024-11-21 15:10:23 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

REST API: add events search

Browse source
This commit is contained in:
Nicola Murino 2021-10-23 15:47:21 +02:00
parent 97d0a48557
commit 74fc3aaf37
No known key found for this signature in database
GPG key ID: 2F1FB59433D5A8CB
25 changed files with 1708 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
.github/workflows/development.yml vendored
View file

@ -31,7 +31,11 @@ jobs:
- name: Build for Linux/macOS x86_64
if: startsWith(matrix.os, 'windows-') != true
run: go build -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/version.commit=`git describe --always --dirty` -X github.com/drakkan/sftpgo/v2/version.date=`date -u +%FT%TZ`" -o sftpgo
run: |
go build -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/version.commit=`git describe --always --dirty` -X github.com/drakkan/sftpgo/v2/version.date=`date -u +%FT%TZ`" -o sftpgo
cd tests/eventsearcher
go build -trimpath -ldflags "-s -w" -o eventsearcher
cd -
- name: Build for macOS arm64
if: startsWith(matrix.os, 'macos-') == true
@ -43,6 +47,9 @@ jobs:
$GIT_COMMIT = (git describe --always --dirty) | Out-String
$DATE_TIME = ([datetime]::Now.ToUniversalTime().toString("yyyy-MM-ddTHH:mm:ssZ")) | Out-String
go build -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/version.commit=$GIT_COMMIT -X github.com/drakkan/sftpgo/v2/version.date=$DATE_TIME" -o sftpgo.exe
cd tests/eventsearcher
go build -trimpath -ldflags "-s -w" -o eventsearcher.exe
cd ../..
- name: Run test cases using SQLite provider
run: go test -v -p 1 -timeout 15m ./... -coverprofile=coverage.txt -covermode=atomic
@ -120,7 +127,10 @@ jobs:
go-version: 1.17
- name: Build
run: go build -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/version.commit=`git describe --always --dirty` -X github.com/drakkan/sftpgo/v2/version.date=`date -u +%FT%TZ`" -o sftpgo
run: |
cd tests/eventsearcher
go build -trimpath -ldflags "-s -w" -o eventsearcher
cd -
env:
GOARCH: 386
@ -173,7 +183,10 @@ jobs:
go-version: 1.17
- name: Build
run: go build -trimpath -ldflags "-s -w -X github.com/drakkan/sftpgo/v2/version.commit=`git describe --always --dirty` -X github.com/drakkan/sftpgo/v2/version.date=`date -u +%FT%TZ`" -o sftpgo
run: |
cd tests/eventsearcher
go build -trimpath -ldflags "-s -w" -o eventsearcher
cd -
- name: Run tests using PostgreSQL provider
run: |

3
.golangci.yml
View file

@ -40,4 +40,5 @@ linters:
- whitespace
- dupl
- rowserrcheck
- dogsled
- dogsled
- govet

5
config/config.go
View file

@ -422,6 +422,11 @@ func GetPluginsConfig() []plugin.Config {
return globalConf.PluginsConfig
}
// SetPluginsConfig sets the plugin configuration
func SetPluginsConfig(config []plugin.Config) {
globalConf.PluginsConfig = config
}
// GetMFAConfig returns multi-factor authentication config
func GetMFAConfig() mfa.Config {
return globalConf.MFAConfig

10
config/config_test.go
View file

@ -19,6 +19,7 @@ import (
"github.com/drakkan/sftpgo/v2/httpd"
"github.com/drakkan/sftpgo/v2/kms"
"github.com/drakkan/sftpgo/v2/mfa"
"github.com/drakkan/sftpgo/v2/sdk/plugin"
"github.com/drakkan/sftpgo/v2/sftpd"
"github.com/drakkan/sftpgo/v2/smtp"
"github.com/drakkan/sftpgo/v2/util"
@ -292,6 +293,15 @@ func TestSetGetConfig(t *testing.T) {
config.SetTelemetryConfig(telemetryConf)
assert.Equal(t, telemetryConf.BindPort, config.GetTelemetryConfig().BindPort)
assert.Equal(t, telemetryConf.BindAddress, config.GetTelemetryConfig().BindAddress)
pluginConf := []plugin.Config{
{
Type: "eventsearcher",
},
}
config.SetPluginsConfig(pluginConf)
if assert.Len(t, config.GetPluginsConfig(), 1) {
assert.Equal(t, pluginConf[0].Type, config.GetPluginsConfig()[0].Type)
}
}
func TestServiceToStart(t *testing.T) {

3
dataprovider/admin.go
View file

@ -39,6 +39,7 @@ const (
PermAdminManageDefender = "manage_defender"
PermAdminViewDefender = "view_defender"
PermAdminRetentionChecks = "retention_checks"
PermAdminViewEvents = "view_events"
)
var (
@ -46,7 +47,7 @@ var (
validAdminPerms = []string{PermAdminAny, PermAdminAddUsers, PermAdminChangeUsers, PermAdminDeleteUsers,
PermAdminViewUsers, PermAdminViewConnections, PermAdminCloseConnections, PermAdminViewServerStatus,
PermAdminManageAdmins, PermAdminManageAPIKeys, PermAdminQuotaScans, PermAdminManageSystem,
PermAdminManageDefender, PermAdminViewDefender, PermAdminRetentionChecks}
PermAdminManageDefender, PermAdminViewDefender, PermAdminRetentionChecks, PermAdminViewEvents}
)
// TOTPConfig defines the time-based one time password configuration

2
docs/custom-actions.md
View file

@ -111,4 +111,4 @@ You can forward SFTPGo events to several publish/subscribe systems using the [sf
## Database services
You can store SFTPGo events in database systems using the [sftpgo-plugin-eventstore](https://github.com/sftpgo/sftpgo-plugin-eventstore).
You can store SFTPGo events in database systems using the [sftpgo-plugin-eventstore](https://github.com/sftpgo/sftpgo-plugin-eventstore) and you can search the stored events using the [sftpgo-plugin-eventsearch](https://github.com/sftpgo/sftpgo-plugin-eventsearch).

1
docs/rest-api.md
View file

@ -36,6 +36,7 @@ You can create other administrator and assign them the following permissions:
- manage system
- manage admins
- manage data retention
- view events
You can also restrict administrator access based on the source IP address. If you are running SFTPGo behind a reverse proxy you need to allow both the proxy IP address and the real client IP.

10
go.mod
View file

@ -7,7 +7,7 @@ require (
github.com/Azure/azure-storage-blob-go v0.14.0
github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962
github.com/alexedwards/argon2id v0.0.0-20210511081203-7d35d68092b8
github.com/aws/aws-sdk-go v1.41.6
github.com/aws/aws-sdk-go v1.41.9
github.com/cockroachdb/cockroach-go/v2 v2.2.1
github.com/eikenb/pipeat v0.0.0-20210603033007-44fc3ffce52b
github.com/fatih/color v1.13.0 // indirect
@ -29,7 +29,7 @@ require (
github.com/klauspost/compress v1.13.6
github.com/kr/text v0.2.0 // indirect
github.com/lestrrat-go/backoff/v2 v2.0.8 // indirect
github.com/lestrrat-go/jwx v1.2.7
github.com/lestrrat-go/jwx v1.2.8
github.com/lib/pq v1.10.3
github.com/lithammer/shortuuid/v3 v3.0.7
github.com/mattn/go-isatty v0.0.14 // indirect
@ -44,7 +44,7 @@ require (
github.com/pkg/sftp v1.13.4
github.com/pquerna/otp v1.3.0
github.com/prometheus/client_golang v1.11.0
github.com/prometheus/common v0.32.0 // indirect
github.com/prometheus/common v0.32.1 // indirect
github.com/rs/cors v1.8.0
github.com/rs/xid v1.3.0
github.com/rs/zerolog v1.25.0
@ -63,10 +63,10 @@ require (
gocloud.dev v0.24.0
golang.org/x/crypto v0.0.0-20210915214749-c084706c2272
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f
golang.org/x/sys v0.0.0-20211020154033-fcb26fe61c20
golang.org/x/sys v0.0.0-20211023085530-d6a326fbbf70
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac
google.golang.org/api v0.59.0
google.golang.org/genproto v0.0.0-20211020151524-b7c3a969101a // indirect
google.golang.org/genproto v0.0.0-20211021150943-2b146023228c // indirect
google.golang.org/grpc v1.41.0
google.golang.org/protobuf v1.27.1
gopkg.in/natefinch/lumberjack.v2 v2.0.0

22
go.sum
View file

@ -137,8 +137,8 @@ github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZo
github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/aws/aws-sdk-go v1.38.68/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/aws/aws-sdk-go v1.40.34/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
github.com/aws/aws-sdk-go v1.41.6 h1:ojO1jWhE3lkJlTFQOq0rlWZ11q18LIdsZNtGJ07FFEA=
github.com/aws/aws-sdk-go v1.41.6/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
github.com/aws/aws-sdk-go v1.41.9 h1:Xb4gWjA90ju0u6Fr2lMAsMOGuhw1g4sTFOqh9SUHgN0=
github.com/aws/aws-sdk-go v1.41.9/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
github.com/aws/aws-sdk-go-v2 v1.7.0/go.mod h1:tb9wi5s61kTDA5qCkcDbt3KRVV74GGslQkl/DRdX/P4=
github.com/aws/aws-sdk-go-v2 v1.9.0/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4=
github.com/aws/aws-sdk-go-v2/config v1.7.0/go.mod h1:w9+nMZ7soXCe5nT46Ri354SNhXDQ6v+V5wqDjnZE+GY=
@ -290,7 +290,6 @@ github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22
github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
github.com/goccy/go-json v0.7.6/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/goccy/go-json v0.7.8/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/goccy/go-json v0.7.10 h1:ulhbuNe1JqE68nMRXXTJRrUu0uhouf0VevLINxQq4Ec=
github.com/goccy/go-json v0.7.10/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
@ -549,14 +548,13 @@ github.com/lestrrat-go/backoff/v2 v2.0.8/go.mod h1:rHP/q/r9aT27n24JQLa7JhSQZCKBB
github.com/lestrrat-go/blackmagic v1.0.0 h1:XzdxDbuQTz0RZZEmdU7cnQxUtFUzgCSPq8RCz4BxIi4=
github.com/lestrrat-go/blackmagic v1.0.0/go.mod h1:TNgH//0vYSs8VXDCfkZLgIrVTTXQELZffUV0tz3MtdQ=
github.com/lestrrat-go/codegen v1.0.1/go.mod h1:JhJw6OQAuPEfVKUCLItpaVLumDGWQznd1VaXrBk9TdM=
github.com/lestrrat-go/codegen v1.0.2/go.mod h1:JhJw6OQAuPEfVKUCLItpaVLumDGWQznd1VaXrBk9TdM=
github.com/lestrrat-go/httpcc v1.0.0 h1:FszVC6cKfDvBKcJv646+lkh4GydQg2Z29scgUfkOpYc=
github.com/lestrrat-go/httpcc v1.0.0/go.mod h1:tGS/u00Vh5N6FHNkExqGGNId8e0Big+++0Gf8MBnAvE=
github.com/lestrrat-go/iter v1.0.1 h1:q8faalr2dY6o8bV45uwrxq12bRa1ezKrB6oM9FUgN4A=
github.com/lestrrat-go/iter v1.0.1/go.mod h1:zIdgO1mRKhn8l9vrZJZz9TUMMFbQbLeTsbqPDrJ/OJc=
github.com/lestrrat-go/jwx v1.2.6/go.mod h1:tJuGuAI3LC71IicTx82Mz1n3w9woAs2bYJZpkjJQ5aU=
github.com/lestrrat-go/jwx v1.2.7 h1:wO7fEc3PW56wpQBMU5CyRkrk4DVsXxCoJg7oIm5HHE4=
github.com/lestrrat-go/jwx v1.2.7/go.mod h1:bw24IXWbavc0R2RsOtpXL7RtMyP589yZ1+L7kd09ZGA=
github.com/lestrrat-go/jwx v1.2.8 h1:qvSpsYZrI5gyFUnb6cmaEqef470/glBiz2ADEDFlyT0=
github.com/lestrrat-go/jwx v1.2.8/go.mod h1:25DcLbNWArPA/Ew5CcBmewl32cJKxOk5cbepBsIJFzw=
github.com/lestrrat-go/option v1.0.0 h1:WqAWL8kh8VcSoD6xjSH34/1m8yxluXQbDeKNfvFeEO4=
github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
@ -688,8 +686,8 @@ github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6T
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
github.com/prometheus/common v0.32.0 h1:HRmM4uANZDAjdvbsdfOoqI5UDbjz0faKeMs/cGPKKI0=
github.com/prometheus/common v0.32.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4=
github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
@ -965,8 +963,8 @@ golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211020154033-fcb26fe61c20 h1:NPtIbR2t8Mhg6UCbkTQMNMejkPpH6IvV4PdZnh1Mqpk=
golang.org/x/sys v0.0.0-20211020154033-fcb26fe61c20/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211023085530-d6a326fbbf70 h1:SeSEfdIxyvwGJliREIJhRPPXvW6sDlLT+UQ3B0hD0NA=
golang.org/x/sys v0.0.0-20211023085530-d6a326fbbf70/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@ -1169,8 +1167,8 @@ google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4/go.mod h1:eFjDcFEc
google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
google.golang.org/genproto v0.0.0-20211008145708-270636b82663/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
google.golang.org/genproto v0.0.0-20211016002631-37fc39342514/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
google.golang.org/genproto v0.0.0-20211020151524-b7c3a969101a h1:8maMHMQp9NroHXhc3HelFX9Ay2lWlXLcdH5mw5Biz0s=
google.golang.org/genproto v0.0.0-20211020151524-b7c3a969101a/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
google.golang.org/genproto v0.0.0-20211021150943-2b146023228c h1:FqrtZMB5Wr+/RecOM3uPJNPfWR8Upb5hAPnt7PU6i4k=
google.golang.org/genproto v0.0.0-20211021150943-2b146023228c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=

154
httpd/api_events.go Normal file
View file

@ -0,0 +1,154 @@
package httpd
import (
"fmt"
"net/http"
"strconv"
"github.com/drakkan/sftpgo/v2/dataprovider"
"github.com/drakkan/sftpgo/v2/sdk/plugin"
"github.com/drakkan/sftpgo/v2/util"
)
type commonEventSearchParams struct {
StartTimestamp int64
EndTimestamp int64
Actions []string
Username string
IP string
InstanceIDs []string
ExcludeIDs []string
Limit int
Order int
}
func (c *commonEventSearchParams) fromRequest(r *http.Request) error {
c.Limit = 100
if _, ok := r.URL.Query()["limit"]; ok {
limit, err := strconv.Atoi(r.URL.Query().Get("limit"))
if err != nil {
return util.NewValidationError(fmt.Sprintf("invalid limit: %v", err))
}
if limit < 1 || limit > 1000 {
return util.NewValidationError(fmt.Sprintf("limit is out of the 1-1000 range: %v", limit))
}
c.Limit = limit
}
if _, ok := r.URL.Query()["order"]; ok {
order := r.URL.Query().Get("order")
if order != dataprovider.OrderASC && order != dataprovider.OrderDESC {
return util.NewValidationError(fmt.Sprintf("invalid order %#v", order))
}
if order == dataprovider.OrderASC {
c.Order = 1
}
}
if _, ok := r.URL.Query()["start_timestamp"]; ok {
ts, err := strconv.ParseInt(r.URL.Query().Get("start_timestamp"), 10, 64)
if err != nil {
return util.NewValidationError(fmt.Sprintf("invalid start_timestamp: %v", err))
}
c.StartTimestamp = ts
}
if _, ok := r.URL.Query()["end_timestamp"]; ok {
ts, err := strconv.ParseInt(r.URL.Query().Get("end_timestamp"), 10, 64)
if err != nil {
return util.NewValidationError(fmt.Sprintf("invalid end_timestamp: %v", err))
}
c.EndTimestamp = ts
}
c.Actions = getCommaSeparatedQueryParam(r, "actions")
c.Username = r.URL.Query().Get("username")
c.IP = r.URL.Query().Get("ip")
c.InstanceIDs = getCommaSeparatedQueryParam(r, "instance_ids")
c.ExcludeIDs = getCommaSeparatedQueryParam(r, "exclude_ids")
return nil
}
type fsEventSearchParams struct {
commonEventSearchParams
SSHCmd string
Protocols []string
Statuses []int32
}
func (s *fsEventSearchParams) fromRequest(r *http.Request) error {
if err := s.commonEventSearchParams.fromRequest(r); err != nil {
return err
}
s.IP = r.URL.Query().Get("ip")
s.SSHCmd = r.URL.Query().Get("ssh_cmd")
s.Protocols = getCommaSeparatedQueryParam(r, "protocols")
statuses := getCommaSeparatedQueryParam(r, "statuses")
for _, status := range statuses {
val, err := strconv.Atoi(status)
if err != nil {
return util.NewValidationError(fmt.Sprintf("invalid status: %v", status))
}
s.Statuses = append(s.Statuses, int32(val))
}
return nil
}
type providerEventSearchParams struct {
commonEventSearchParams
ObjectName string
ObjectTypes []string
}
func (s *providerEventSearchParams) fromRequest(r *http.Request) error {
if err := s.commonEventSearchParams.fromRequest(r); err != nil {
return err
}
s.ObjectName = r.URL.Query().Get("object_name")
s.ObjectTypes = getCommaSeparatedQueryParam(r, "object_types")
return nil
}
func searchFsEvents(w http.ResponseWriter, r *http.Request) {
r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize)
params := fsEventSearchParams{}
err := params.fromRequest(r)
if err != nil {
sendAPIResponse(w, r, err, "", getRespStatus(err))
return
}
data, _, _, err := plugin.Handler.SearchFsEvents(params.StartTimestamp, params.EndTimestamp, params.Username,
params.IP, params.SSHCmd, params.Actions, params.Protocols, params.InstanceIDs, params.ExcludeIDs,
params.Statuses, params.Limit, params.Order)
if err != nil {
sendAPIResponse(w, r, err, "", getRespStatus(err))
return
}
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.Write(data) //nolint:errcheck
}
func searchProviderEvents(w http.ResponseWriter, r *http.Request) {
r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize)
params := providerEventSearchParams{}
err := params.fromRequest(r)
if err != nil {
sendAPIResponse(w, r, err, "", getRespStatus(err))
return
}
data, _, _, err := plugin.Handler.SearchProviderEvents(params.StartTimestamp, params.EndTimestamp, params.Username,
params.IP, params.ObjectName, params.Limit, params.Order, params.Actions, params.ObjectTypes, params.InstanceIDs,
params.ExcludeIDs)
if err != nil {
sendAPIResponse(w, r, err, "", getRespStatus(err))
return
}
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.Write(data) //nolint:errcheck
}

12
httpd/api_retention.go
View file

@ -3,13 +3,11 @@ package httpd
import (
"fmt"
"net/http"
"strings"
"github.com/go-chi/render"
"github.com/drakkan/sftpgo/v2/common"
"github.com/drakkan/sftpgo/v2/dataprovider"
"github.com/drakkan/sftpgo/v2/util"
)
func getRetentionChecks(w http.ResponseWriter, r *http.Request) {
@ -26,18 +24,14 @@ func startRetentionCheck(w http.ResponseWriter, r *http.Request) {
return
}
var check common.RetentionCheck
err = render.DecodeJSON(r.Body, &check.Folders)
if err != nil {
sendAPIResponse(w, r, err, "", http.StatusBadRequest)
return
}
for _, val := range strings.Split(r.URL.Query().Get("notifications"), ",") {
val = strings.TrimSpace(val)
if val != "" {
check.Notifications = append(check.Notifications, val)
}
}
check.Notifications = util.RemoveDuplicates(check.Notifications)
check.Notifications = getCommaSeparatedQueryParam(r, "notifications")
for _, notification := range check.Notifications {
if notification == common.RetentionCheckNotificationEmail {
claims, err := getTokenClaims(r)

28
httpd/api_utils.go
View file

@ -7,12 +7,14 @@ import (
"io"
"mime"
"net/http"
"net/url"
"os"
"path"
"strconv"
"strings"
"time"
"github.com/go-chi/chi/v5"
"github.com/go-chi/render"
"github.com/klauspost/compress/zip"
@ -20,6 +22,7 @@ import (
"github.com/drakkan/sftpgo/v2/dataprovider"
"github.com/drakkan/sftpgo/v2/logger"
"github.com/drakkan/sftpgo/v2/metric"
"github.com/drakkan/sftpgo/v2/sdk/plugin"
"github.com/drakkan/sftpgo/v2/util"
)
@ -74,6 +77,9 @@ func getRespStatus(err error) int {
if os.IsPermission(err) {
return http.StatusForbidden
}
if errors.Is(err, plugin.ErrNoSearcher) {
return http.StatusNotImplemented
}
return http.StatusInternalServerError
}
@ -90,6 +96,28 @@ func getMappedStatusCode(err error) int {
return statusCode
}
func getURLParam(r *http.Request, key string) string {
v := chi.URLParam(r, key)
unescaped, err := url.PathUnescape(v)
if err != nil {
return v
}
return unescaped
}
func getCommaSeparatedQueryParam(r *http.Request, key string) []string {
var result []string
for _, val := range strings.Split(r.URL.Query().Get(key), ",") {
val = strings.TrimSpace(val)
if val != "" {
result = append(result, val)
}
}
return util.RemoveDuplicates(result)
}
func handleCloseConnection(w http.ResponseWriter, r *http.Request) {
r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize)
connectionID := getURLParam(r, "connectionID")

12
httpd/httpd.go
View file

@ -8,7 +8,6 @@ import (
"fmt"
"net"
"net/http"
"net/url"
"path"
"path/filepath"
"runtime"
@ -77,6 +76,8 @@ const (
userProfilePath = "/api/v2/user/profile"
retentionBasePath = "/api/v2/retention/users"
retentionChecksPath = "/api/v2/retention/users/checks"
fsEventsPath = "/api/v2/events/fs"
providerEventsPath = "/api/v2/events/provider"
healthzPath = "/healthz"
webRootPathDefault = "/"
webBasePathDefault = "/web"
@ -490,15 +491,6 @@ func getServicesStatus() ServicesStatus {
return status
}
func getURLParam(r *http.Request, key string) string {
v := chi.URLParam(r, key)
unescaped, err := url.PathUnescape(v)
if err != nil {
return v
}
return unescaped
}
func fileServer(r chi.Router, path string, root http.FileSystem) {
if path != "/" && path[len(path)-1] != '/' {
r.Get(path, http.RedirectHandler(path+"/", http.StatusMovedPermanently).ServeHTTP)

90
httpd/httpd_test.go
View file

@ -46,6 +46,7 @@ import (
"github.com/drakkan/sftpgo/v2/logger"
"github.com/drakkan/sftpgo/v2/mfa"
"github.com/drakkan/sftpgo/v2/sdk"
"github.com/drakkan/sftpgo/v2/sdk/plugin"
"github.com/drakkan/sftpgo/v2/sftpd"
"github.com/drakkan/sftpgo/v2/util"
"github.com/drakkan/sftpgo/v2/vfs"
@ -100,6 +101,8 @@ const (
user2FARecoveryCodesPath = "/api/v2/user/2fa/recoverycodes"
userProfilePath = "/api/v2/user/profile"
retentionBasePath = "/api/v2/retention/users"
fsEventsPath = "/api/v2/events/fs"
providerEventsPath = "/api/v2/events/provider"
healthzPath = "/healthz"
webBasePath = "/web"
webBasePathAdmin = "/web/admin"
@ -248,6 +251,21 @@ func TestMain(m *testing.M) {
logger.WarnToConsole("error loading configuration: %v", err)
os.Exit(1)
}
wdPath, err := os.Getwd()
if err != nil {
logger.WarnToConsole("error getting exe path: %v", err)
os.Exit(1)
}
pluginsConfig := []plugin.Config{
{
Type: "eventsearcher",
Cmd: filepath.Join(wdPath, "..", "tests", "eventsearcher", "eventsearcher"),
AutoMTLS: true,
},
}
if runtime.GOOS == osWindows {
pluginsConfig[0].Cmd += ".exe"
}
providerConf := config.GetProviderConf()
credentialsPath = filepath.Join(os.TempDir(), "test_credentials")
providerConf.CredentialsPath = credentialsPath
@ -284,6 +302,11 @@ func TestMain(m *testing.M) {
logger.ErrorToConsole("error initializing MFA: %v", err)
os.Exit(1)
}
err = plugin.Initialize(pluginsConfig, true)
if err != nil {
logger.ErrorToConsole("error initializing plugin: %v", err)
os.Exit(1)
}
httpdConf := config.GetHTTPDConfig()
@ -5512,6 +5535,73 @@ func TestWebUserTwoFactorLogin(t *testing.T) {
checkResponseCode(t, http.StatusInternalServerError, rr)
}
func TestSearchEvents(t *testing.T) {
token, err := getJWTAPITokenFromTestServer(defaultTokenAuthUser, defaultTokenAuthPass)
assert.NoError(t, err)
req, err := http.NewRequest(http.MethodGet, fsEventsPath+"?limit=10&order=ASC", nil)
assert.NoError(t, err)
setBearerForReq(req, token)
rr := executeRequest(req)
checkResponseCode(t, http.StatusOK, rr)
// the test eventsearcher plugin returns error if start_timestamp < 0
req, err = http.NewRequest(http.MethodGet, fsEventsPath+"?start_timestamp=-1&end_timestamp=123456&statuses=1,2", nil)
assert.NoError(t, err)
setBearerForReq(req, token)
rr = executeRequest(req)
checkResponseCode(t, http.StatusInternalServerError, rr)
req, err = http.NewRequest(http.MethodGet, fsEventsPath+"?limit=a", nil)
assert.NoError(t, err)
setBearerForReq(req, token)
rr = executeRequest(req)
checkResponseCode(t, http.StatusBadRequest, rr)
req, err = http.NewRequest(http.MethodGet, providerEventsPath, nil)
assert.NoError(t, err)
setBearerForReq(req, token)
rr = executeRequest(req)
checkResponseCode(t, http.StatusOK, rr)
// the test eventsearcher plugin returns error if start_timestamp < 0
req, err = http.NewRequest(http.MethodGet, providerEventsPath+"?start_timestamp=-1", nil)
assert.NoError(t, err)
setBearerForReq(req, token)
rr = executeRequest(req)
checkResponseCode(t, http.StatusInternalServerError, rr)
req, err = http.NewRequest(http.MethodGet, providerEventsPath+"?limit=2000", nil)
assert.NoError(t, err)
setBearerForReq(req, token)
rr = executeRequest(req)
checkResponseCode(t, http.StatusBadRequest, rr)
req, err = http.NewRequest(http.MethodGet, fsEventsPath+"?start_timestamp=a", nil)
assert.NoError(t, err)
setBearerForReq(req, token)
rr = executeRequest(req)
checkResponseCode(t, http.StatusBadRequest, rr)
req, err = http.NewRequest(http.MethodGet, fsEventsPath+"?end_timestamp=a", nil)
assert.NoError(t, err)
setBearerForReq(req, token)
rr = executeRequest(req)
checkResponseCode(t, http.StatusBadRequest, rr)
req, err = http.NewRequest(http.MethodGet, fsEventsPath+"?order=ASSC", nil)
assert.NoError(t, err)
setBearerForReq(req, token)
rr = executeRequest(req)
checkResponseCode(t, http.StatusBadRequest, rr)
req, err = http.NewRequest(http.MethodGet, fsEventsPath+"?statuses=a,b", nil)
assert.NoError(t, err)
setBearerForReq(req, token)
rr = executeRequest(req)
checkResponseCode(t, http.StatusBadRequest, rr)
}
func TestMFAErrors(t *testing.T) {
user, _, err := httpdtest.AddUser(getTestUser(), http.StatusCreated)
assert.NoError(t, err)

3
httpd/internal_test.go
View file

@ -33,6 +33,7 @@ import (
"github.com/drakkan/sftpgo/v2/dataprovider"
"github.com/drakkan/sftpgo/v2/kms"
"github.com/drakkan/sftpgo/v2/sdk"
"github.com/drakkan/sftpgo/v2/sdk/plugin"
"github.com/drakkan/sftpgo/v2/util"
"github.com/drakkan/sftpgo/v2/vfs"
)
@ -305,6 +306,8 @@ func TestGetRespStatus(t *testing.T) {
err = fmt.Errorf("generic error")
respStatus = getRespStatus(err)
assert.Equal(t, http.StatusInternalServerError, respStatus)
respStatus = getRespStatus(plugin.ErrNoSearcher)
assert.Equal(t, http.StatusNotImplemented, respStatus)
}
func TestMappedStatusCode(t *testing.T) {

2
httpd/schema/openapi.yaml
View file

@ -3400,6 +3400,7 @@ components:
- manage_defender
- view_defender
- retention_checks
- view_events
description: |
Admin permissions:
* `*` - all permissions are granted
@ -3417,6 +3418,7 @@ components:
* `manage_defender` - remove ip from the dynamic blocklist is allowed
* `view_defender` - list the dynamic blocklist is allowed
* `retention_checks` - view and start retention checks is allowed
* `view_events` - view and search filesystem and provider events is allowed
LoginMethods:
type: string
enum:

4
httpd/server.go
View file

@ -977,6 +977,10 @@ func (s *httpdServer) initializeRouter() {
router.With(checkPerm(dataprovider.PermAdminRetentionChecks)).Get(retentionChecksPath, getRetentionChecks)
router.With(checkPerm(dataprovider.PermAdminRetentionChecks)).Post(retentionBasePath+"/{username}/check",
startRetentionCheck)
router.With(checkPerm(dataprovider.PermAdminViewEvents), compressor.Handler).
Get(fsEventsPath, searchFsEvents)
router.With(checkPerm(dataprovider.PermAdminViewEvents), compressor.Handler).
Get(providerEventsPath, searchProviderEvents)
router.With(forbidAPIKeyAuthentication, checkPerm(dataprovider.PermAdminManageAPIKeys)).
Get(apiKeysPath, getAPIKeys)
router.With(forbidAPIKeyAuthentication, checkPerm(dataprovider.PermAdminManageAPIKeys)).

3
sdk/plugin/auth.go
View file

@ -5,7 +5,6 @@ import (
"errors"
"fmt"
"os/exec"
"path/filepath"
"github.com/hashicorp/go-hclog"
"github.com/hashicorp/go-plugin"
@ -117,7 +116,7 @@ func (p *authPlugin) initialize() error {
Managed: false,
Logger: &logger.HCLogAdapter{
Logger: hclog.New(&hclog.LoggerOptions{
Name: fmt.Sprintf("%v.%v.%v", logSender, auth.PluginName, filepath.Base(p.config.Cmd)),
Name: fmt.Sprintf("%v.%v", logSender, auth.PluginName),
Level: pluginsLogLevel,
DisableTime: true,
}),

12
sdk/plugin/eventsearcher/grpc.go
View file

@ -19,7 +19,7 @@ type GRPCClient struct {
// SearchFsEvents implements the Searcher interface
func (c *GRPCClient) SearchFsEvents(startTimestamp, endTimestamp int64, username, ip, sshCmd string, actions,
protocols, instanceIDs, excludeIDs []string, statuses []int32, limit, order int,
) ([]byte, error) {
) ([]byte, []string, []string, error) {
ctx, cancel := context.WithTimeout(context.Background(), rpcTimeout)
defer cancel()
@ -39,15 +39,15 @@ func (c *GRPCClient) SearchFsEvents(startTimestamp, endTimestamp int64, username
})
if err != nil {
return nil, err
return nil, nil, nil, err
}
return resp.Data, nil
return resp.Data, resp.SameTsAtStart, resp.SameTsAtEnd, nil
}
// SearchProviderEvents implements the Searcher interface
func (c *GRPCClient) SearchProviderEvents(startTimestamp, endTimestamp int64, username, ip, objectName string,
limit, order int, actions, objectTypes, instanceIDs, excludeIDs []string,
) ([]byte, error) {
) ([]byte, []string, []string, error) {
ctx, cancel := context.WithTimeout(context.Background(), rpcTimeout)
defer cancel()
@ -66,9 +66,9 @@ func (c *GRPCClient) SearchProviderEvents(startTimestamp, endTimestamp int64, us
})
if err != nil {
return nil, err
return nil, nil, nil, err
}
return resp.Data, nil
return resp.Data, resp.SameTsAtStart, resp.SameTsAtEnd, nil
}
// GRPCServer defines the gRPC server that GRPCClient talks to.

2
sdk/plugin/kms.go
View file

@ -77,7 +77,7 @@ func (p *kmsPlugin) initialize() error {
Managed: false,
Logger: &logger.HCLogAdapter{
Logger: hclog.New(&hclog.LoggerOptions{
Name: fmt.Sprintf("%v.%v.%v", logSender, kmsplugin.PluginName, filepath.Base(p.config.Cmd)),
Name: fmt.Sprintf("%v.%v", logSender, kmsplugin.PluginName),
Level: pluginsLogLevel,
DisableTime: true,
}),

3
sdk/plugin/notifier.go
View file

@ -4,7 +4,6 @@ import (
"crypto/sha256"
"fmt"
"os/exec"
"path/filepath"
"sync"
"time"
@ -166,7 +165,7 @@ func (p *notifierPlugin) initialize() error {
Managed: false,
Logger: &logger.HCLogAdapter{
Logger: hclog.New(&hclog.LoggerOptions{
Name: fmt.Sprintf("%v.%v.%v", logSender, notifier.PluginName, filepath.Base(p.config.Cmd)),
Name: fmt.Sprintf("%v.%v", logSender, notifier.PluginName),
Level: pluginsLogLevel,
DisableTime: true,
}),

3
sdk/plugin/searcher.go
View file

@ -4,7 +4,6 @@ import (
"crypto/sha256"
"fmt"
"os/exec"
"path/filepath"
"github.com/hashicorp/go-hclog"
"github.com/hashicorp/go-plugin"
@ -58,7 +57,7 @@ func (p *searcherPlugin) initialize() error {
Managed: false,
Logger: &logger.HCLogAdapter{
Logger: hclog.New(&hclog.LoggerOptions{
Name: fmt.Sprintf("%v.%v.%v", logSender, eventsearcher.PluginName, filepath.Base(p.config.Cmd)),
Name: fmt.Sprintf("%v.%v", logSender, eventsearcher.PluginName),
Level: pluginsLogLevel,
DisableTime: true,
}),

27
tests/eventsearcher/go.mod Normal file
View file

@ -0,0 +1,27 @@
module github.com/drakkan/sftpgo/tests/eventsearcher
go 1.17
require (
github.com/drakkan/sftpgo/v2 v2.1.1-0.20211020173949-97d0a4855756
github.com/hashicorp/go-plugin v1.4.3
)
require (
github.com/fatih/color v1.13.0 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/hashicorp/go-hclog v1.0.0 // indirect
github.com/hashicorp/yamux v0.0.0-20210826001029-26ff87cf9493 // indirect
github.com/mattn/go-colorable v0.1.11 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/mitchellh/go-testing-interface v1.14.1 // indirect
github.com/oklog/run v1.1.0 // indirect
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f // indirect
golang.org/x/sys v0.0.0-20211023085530-d6a326fbbf70 // indirect
golang.org/x/text v0.3.7 // indirect
google.golang.org/genproto v0.0.0-20211021150943-2b146023228c // indirect
google.golang.org/grpc v1.41.0 // indirect
google.golang.org/protobuf v1.27.1 // indirect
)
replace github.com/drakkan/sftpgo/v2 => ../..

1216
tests/eventsearcher/go.sum Normal file
View file

File diff suppressed because it is too large Load diff

117
tests/eventsearcher/main.go Normal file
View file

@ -0,0 +1,117 @@
package main
import (
"encoding/json"
"errors"
"github.com/drakkan/sftpgo/v2/sdk/plugin/eventsearcher"
"github.com/hashicorp/go-plugin"
)
var (
errNotSupported = errors.New("unsupported parameter")
)
type fsEvent struct {
ID string `json:"id"`
Timestamp int64 `json:"timestamp"`
Action string `json:"action"`
Username string `json:"username"`
FsPath string `json:"fs_path"`
FsTargetPath string `json:"fs_target_path,omitempty"`
VirtualPath string `json:"virtual_path"`
VirtualTargetPath string `json:"virtual_target_path,omitempty"`
SSHCmd string `json:"ssh_cmd,omitempty"`
FileSize int64 `json:"file_size,omitempty"`
Status int `json:"status"`
Protocol string `json:"protocol"`
IP string `json:"ip,omitempty"`
InstanceID string `json:"instance_id,omitempty"`
}
type providerEvent struct {
ID string `json:"id" gorm:"primaryKey"`
Timestamp int64 `json:"timestamp"`
Action string `json:"action"`
Username string `json:"username"`
IP string `json:"ip,omitempty"`
ObjectType string `json:"object_type"`
ObjectName string `json:"object_name"`
ObjectData []byte `json:"object_data"`
InstanceID string `json:"instance_id,omitempty"`
}
type Searcher struct{}
func (s *Searcher) SearchFsEvents(startTimestamp, endTimestamp int64, username, ip, sshCmd string, actions,
protocols, instanceIDs, excludeIDs []string, statuses []int32, limit, order int,
) ([]byte, []string, []string, error) {
if startTimestamp < 0 {
return nil, nil, nil, errNotSupported
}
results := []fsEvent{
{
ID: "1",
Timestamp: 100,
Action: "upload",
Username: "username1",
FsPath: "/tmp/file.txt",
FsTargetPath: "/tmp/target.txt",
VirtualPath: "file.txt",
VirtualTargetPath: "target.txt",
SSHCmd: "scp",
FileSize: 123,
Status: 1,
Protocol: "SFTP",
IP: "::1",
InstanceID: "instance1",
},
}
data, err := json.Marshal(results)
if err != nil {
return nil, nil, nil, err
}
return data, nil, nil, nil
}
func (s *Searcher) SearchProviderEvents(startTimestamp, endTimestamp int64, username, ip, objectName string,
limit, order int, actions, objectTypes, instanceIDs, excludeIDs []string,
) ([]byte, []string, []string, error) {
if startTimestamp < 0 {
return nil, nil, nil, errNotSupported
}
results := []providerEvent{
{
ID: "1",
Timestamp: 100,
Action: "add",
Username: "username1",
IP: "127.0.0.1",
ObjectType: "api_key",
ObjectName: "123",
ObjectData: []byte("data"),
InstanceID: "instance1",
},
}
data, err := json.Marshal(results)
if err != nil {
return nil, nil, nil, err
}
return data, nil, nil, nil
}
func main() {
plugin.Serve(&plugin.ServeConfig{
HandshakeConfig: eventsearcher.Handshake,
Plugins: map[string]plugin.Plugin{
eventsearcher.PluginName: &eventsearcher.Plugin{Impl: &Searcher{}},
},
GRPCServer: plugin.DefaultGRPCServer,
})
}