diff --git a/common/common.go b/common/common.go index 78e9b66a..c196022c 100644 --- a/common/common.go +++ b/common/common.go @@ -134,6 +134,34 @@ func IsBanned(ip string) bool { return Config.defender.IsBanned(ip) } +// GetDefenderBanTime returns the ban time for the given IP +// or nil if the IP is not banned or the defender is disabled +func GetDefenderBanTime(ip string) *time.Time { + if Config.defender == nil { + return nil + } + + return Config.defender.GetBanTime(ip) +} + +// Unban removes the specified IP address from the banned ones +func Unban(ip string) bool { + if Config.defender == nil { + return false + } + + return Config.defender.Unban(ip) +} + +// GetDefenderScore returns the score for the given IP +func GetDefenderScore(ip string) int { + if Config.defender == nil { + return 0 + } + + return Config.defender.GetScore(ip) +} + // AddDefenderEvent adds the specified defender event for the given IP func AddDefenderEvent(ip string, event HostEvent) { if Config.defender == nil { diff --git a/common/common_test.go b/common/common_test.go index a335f11c..b513566e 100644 --- a/common/common_test.go +++ b/common/common_test.go @@ -238,6 +238,10 @@ func TestDefenderIntegration(t *testing.T) { AddDefenderEvent(ip, HostEventNoLoginTried) assert.False(t, IsBanned(ip)) + assert.Nil(t, GetDefenderBanTime(ip)) + assert.False(t, Unban(ip)) + assert.Equal(t, 0, GetDefenderScore(ip)) + Config.DefenderConfig = DefenderConfig{ Enabled: true, BanTime: 10, @@ -257,8 +261,17 @@ func TestDefenderIntegration(t *testing.T) { AddDefenderEvent(ip, HostEventNoLoginTried) assert.False(t, IsBanned(ip)) + assert.Equal(t, 2, GetDefenderScore(ip)) + assert.False(t, Unban(ip)) + assert.Nil(t, GetDefenderBanTime(ip)) + AddDefenderEvent(ip, HostEventLoginFailed) assert.True(t, IsBanned(ip)) + assert.Equal(t, 0, GetDefenderScore(ip)) + assert.NotNil(t, GetDefenderBanTime(ip)) + assert.True(t, Unban(ip)) + assert.Nil(t, GetDefenderBanTime(ip)) + assert.False(t, Unban(ip)) Config = configCopy } diff --git a/common/defender.go b/common/defender.go index e122c88b..40b8d453 100644 --- a/common/defender.go +++ b/common/defender.go @@ -32,6 +32,7 @@ type Defender interface { IsBanned(ip string) bool GetBanTime(ip string) *time.Time GetScore(ip string) int + Unban(ip string) bool } // DefenderConfig defines the "defender" configuration @@ -201,6 +202,19 @@ func (d *memoryDefender) IsBanned(ip string) bool { return false } +// Unban removes the specified IP address from the banned ones +func (d *memoryDefender) Unban(ip string) bool { + d.Lock() + defer d.Unlock() + + if _, ok := d.banned[ip]; ok { + delete(d.banned, ip) + return true + } + + return false +} + // AddEvent adds an event for the given IP. // This method must be called for clients not yet banned func (d *memoryDefender) AddEvent(ip string, event HostEvent) { diff --git a/common/defender_test.go b/common/defender_test.go index 0cd89643..b44487b5 100644 --- a/common/defender_test.go +++ b/common/defender_test.go @@ -142,6 +142,9 @@ func TestBasicDefender(t *testing.T) { assert.True(t, newBanTime.After(*banTime)) } + assert.True(t, defender.Unban(testIP3)) + assert.False(t, defender.Unban(testIP3)) + err = os.Remove(slFile) assert.NoError(t, err) err = os.Remove(blFile) diff --git a/docs/defender.md b/docs/defender.md index 3bceb172..8bce7e88 100644 --- a/docs/defender.md +++ b/docs/defender.md @@ -17,13 +17,23 @@ And then you can configure: So a host is banned, for `ban_time` minutes, if it has exceeded the defined threshold during the last observation time minutes. +A banned IP has no score, it makes no sense to accumulate host events in memory for an already banned IP address. + If an already banned client tries to log in again its ban time will be incremented based on the `ban_time_increment` configuration. The `ban_time_increment` is calculated as percentage of `ban_time`, so if `ban_time` is 30 minutes and `ban_time_increment` is 50 the host will be banned for additionally 15 minutes. You can specify values greater than 100 for `ban_time_increment`. The `defender` will keep in memory both the host scores and the banned hosts, you can limit the memory usage using the `entries_soft_limit` and `entries_hard_limit` configuration keys. -The `defender` can also load a permanent block and/or safe list of ip addresses/networks from a file: +The REST API allows: + +- to retrieve the score for an IP address +- to retrieve the ban time for an IP address +- to unban an IP address + +We don't return the whole list of the banned IP addresses or all the stored scores because we store them as hash map and iterating over all the keys for an hash map is slow and will slow down new events registration. + +The `defender` can also load a permanent block list and/or a safe list of ip addresses/networks from a file: - `safelist_file`, string. Path to a file with a list of ip addresses and/or networks to never ban. - `blocklist_file`, string. Path to a file with a list of ip addresses and/or networks to always ban. @@ -48,6 +58,6 @@ Here is a small example: } ``` -These list will be loaded in memory for faster lookups. +These list will be loaded in memory for faster lookups. The REST API queries "live" data and not these lists. The `defender` is optimized for fast and time constant lookups however as it keeps all the lists and the entries in memory you should carefully measure the memory requirements for your use case. diff --git a/httpd/api_defender.go b/httpd/api_defender.go new file mode 100644 index 00000000..f6b0197e --- /dev/null +++ b/httpd/api_defender.go @@ -0,0 +1,82 @@ +package httpd + +import ( + "errors" + "fmt" + "net" + "net/http" + "time" + + "github.com/go-chi/render" + + "github.com/drakkan/sftpgo/common" +) + +func getBanTime(w http.ResponseWriter, r *http.Request) { + ip := r.URL.Query().Get("ip") + err := validateIPAddress(ip) + if err != nil { + sendAPIResponse(w, r, err, "", http.StatusBadRequest) + return + } + + banStatus := make(map[string]*string) + + banTime := common.GetDefenderBanTime(ip) + var banTimeString *string + if banTime != nil { + rfc3339String := banTime.UTC().Format(time.RFC3339) + banTimeString = &rfc3339String + } + + banStatus["date_time"] = banTimeString + render.JSON(w, r, banStatus) +} + +func getScore(w http.ResponseWriter, r *http.Request) { + ip := r.URL.Query().Get("ip") + err := validateIPAddress(ip) + if err != nil { + sendAPIResponse(w, r, err, "", http.StatusBadRequest) + return + } + + scoreStatus := make(map[string]int) + scoreStatus["score"] = common.GetDefenderScore(ip) + + render.JSON(w, r, scoreStatus) +} + +func unban(w http.ResponseWriter, r *http.Request) { + r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize) + + var postBody map[string]string + err := render.DecodeJSON(r.Body, &postBody) + if err != nil { + sendAPIResponse(w, r, err, "", http.StatusBadRequest) + return + } + + ip := postBody["ip"] + err = validateIPAddress(ip) + if err != nil { + sendAPIResponse(w, r, err, "", http.StatusBadRequest) + return + } + + if common.Unban(ip) { + sendAPIResponse(w, r, nil, "OK", http.StatusOK) + } else { + sendAPIResponse(w, r, nil, "Not found", http.StatusNotFound) + } +} + +func validateIPAddress(ip string) error { + if ip == "" { + return errors.New("ip address is required") + } + if net.ParseIP(ip) == nil { + return fmt.Errorf("ip address %#v is not valid", ip) + } + return nil +} diff --git a/httpd/api_utils.go b/httpd/api_utils.go index f513a3c3..d4260d86 100644 --- a/httpd/api_utils.go +++ b/httpd/api_utils.go @@ -446,6 +446,69 @@ func GetStatus(expectedStatusCode int) (ServicesStatus, []byte, error) { return response, body, err } +// GetBanTime returns the ban time for the given IP address +func GetBanTime(ip string, expectedStatusCode int) (map[string]interface{}, []byte, error) { + var response map[string]interface{} + var body []byte + url, err := url.Parse(buildURLRelativeToBase(defenderBanTime)) + if err != nil { + return response, body, err + } + q := url.Query() + q.Add("ip", ip) + url.RawQuery = q.Encode() + resp, err := sendHTTPRequest(http.MethodGet, url.String(), nil, "") + if err != nil { + return response, body, err + } + defer resp.Body.Close() + err = checkResponse(resp.StatusCode, expectedStatusCode) + if err == nil && expectedStatusCode == http.StatusOK { + err = render.DecodeJSON(resp.Body, &response) + } else { + body, _ = getResponseBody(resp) + } + return response, body, err +} + +// GetScore returns the score for the given IP address +func GetScore(ip string, expectedStatusCode int) (map[string]interface{}, []byte, error) { + var response map[string]interface{} + var body []byte + url, err := url.Parse(buildURLRelativeToBase(defenderScore)) + if err != nil { + return response, body, err + } + q := url.Query() + q.Add("ip", ip) + url.RawQuery = q.Encode() + resp, err := sendHTTPRequest(http.MethodGet, url.String(), nil, "") + if err != nil { + return response, body, err + } + defer resp.Body.Close() + err = checkResponse(resp.StatusCode, expectedStatusCode) + if err == nil && expectedStatusCode == http.StatusOK { + err = render.DecodeJSON(resp.Body, &response) + } else { + body, _ = getResponseBody(resp) + } + return response, body, err +} + +// UnbanIP unbans the given IP address +func UnbanIP(ip string, expectedStatusCode int) error { + postBody := make(map[string]string) + postBody["ip"] = ip + asJSON, _ := json.Marshal(postBody) + resp, err := sendHTTPRequest(http.MethodPost, buildURLRelativeToBase(defenderUnban), bytes.NewBuffer(asJSON), "") + if err != nil { + return err + } + defer resp.Body.Close() + return checkResponse(resp.StatusCode, expectedStatusCode) +} + // Dumpdata requests a backup to outputFile. // outputFile is relative to the configured backups_path func Dumpdata(outputFile, indent string, expectedStatusCode int) (map[string]interface{}, []byte, error) { diff --git a/httpd/httpd.go b/httpd/httpd.go index 5c77d3d3..2076a24d 100644 --- a/httpd/httpd.go +++ b/httpd/httpd.go @@ -39,6 +39,9 @@ const ( loadDataPath = "/api/v1/loaddata" updateUsedQuotaPath = "/api/v1/quota_update" updateFolderUsedQuotaPath = "/api/v1/folder_quota_update" + defenderBanTime = "/api/v1/defender/ban_time" + defenderUnban = "/api/v1/defender/unban" + defenderScore = "/api/v1/defender/score" metricsPath = "/metrics" webBasePath = "/web" webUsersPath = "/web/users" @@ -61,12 +64,17 @@ var ( certMgr *common.CertManager ) +type defenderStatus struct { + IsActive bool `json:"is_active"` +} + // ServicesStatus keep the state of the running services type ServicesStatus struct { SSH sftpd.ServiceStatus `json:"ssh"` FTP ftpd.ServiceStatus `json:"ftp"` WebDAV webdavd.ServiceStatus `json:"webdav"` DataProvider dataprovider.ProviderStatus `json:"data_provider"` + Defender defenderStatus `json:"defender"` } // Conf httpd daemon configuration @@ -186,6 +194,9 @@ func getServicesStatus() ServicesStatus { FTP: ftpd.GetStatus(), WebDAV: webdavd.GetStatus(), DataProvider: dataprovider.GetProviderStatus(), + Defender: defenderStatus{ + IsActive: common.Config.DefenderConfig.Enabled, + }, } return status } diff --git a/httpd/httpd_test.go b/httpd/httpd_test.go index 941c828e..f949c5a5 100644 --- a/httpd/httpd_test.go +++ b/httpd/httpd_test.go @@ -50,6 +50,7 @@ const ( quotaScanVFolderPath = "/api/v1/folder_quota_scan" updateUsedQuotaPath = "/api/v1/quota_update" updateFolderUsedQuotaPath = "/api/v1/folder_quota_update" + defenderUnban = "/api/v1/defender/unban" versionPath = "/api/v1/version" metricsPath = "/metrics" webBasePath = "/web" @@ -2200,6 +2201,71 @@ func TestDumpdata(t *testing.T) { assert.NoError(t, err) } +func TestDefenderAPI(t *testing.T) { + oldConfig := config.GetCommonConfig() + + cfg := config.GetCommonConfig() + cfg.DefenderConfig.Enabled = true + cfg.DefenderConfig.Threshold = 3 + + err := common.Initialize(cfg) + require.NoError(t, err) + + ip := "::1" + + response, _, err := httpd.GetBanTime(ip, http.StatusOK) + require.NoError(t, err) + banTime, ok := response["date_time"] + require.True(t, ok) + assert.Nil(t, banTime) + + response, _, err = httpd.GetScore(ip, http.StatusOK) + require.NoError(t, err) + score, ok := response["score"] + require.True(t, ok) + assert.Equal(t, float64(0), score) + + err = httpd.UnbanIP(ip, http.StatusNotFound) + require.NoError(t, err) + + common.AddDefenderEvent(ip, common.HostEventNoLoginTried) + response, _, err = httpd.GetScore(ip, http.StatusOK) + require.NoError(t, err) + score, ok = response["score"] + require.True(t, ok) + assert.Equal(t, float64(2), score) + + common.AddDefenderEvent(ip, common.HostEventNoLoginTried) + response, _, err = httpd.GetBanTime(ip, http.StatusOK) + require.NoError(t, err) + banTime, ok = response["date_time"] + require.True(t, ok) + assert.NotNil(t, banTime) + + err = httpd.UnbanIP(ip, http.StatusOK) + require.NoError(t, err) + + err = httpd.UnbanIP(ip, http.StatusNotFound) + require.NoError(t, err) + + err = common.Initialize(oldConfig) + require.NoError(t, err) +} + +func TestDefenderAPIErrors(t *testing.T) { + _, _, err := httpd.GetBanTime("", http.StatusBadRequest) + require.NoError(t, err) + + _, _, err = httpd.GetBanTime("invalid", http.StatusBadRequest) + require.NoError(t, err) + + _, _, err = httpd.GetScore("", http.StatusBadRequest) + require.NoError(t, err) + + err = httpd.UnbanIP("", http.StatusBadRequest) + require.NoError(t, err) +} + func TestLoaddata(t *testing.T) { mappedPath := filepath.Join(os.TempDir(), "restored_folder") user := getTestUser() @@ -2425,6 +2491,12 @@ func TestAddFolderInvalidJsonMock(t *testing.T) { checkResponseCode(t, http.StatusBadRequest, rr.Code) } +func TestUnbanInvalidJsonMock(t *testing.T) { + req, _ := http.NewRequest(http.MethodPost, defenderUnban, bytes.NewBuffer([]byte("invalid json"))) + rr := executeRequest(req) + checkResponseCode(t, http.StatusBadRequest, rr.Code) +} + func TestAddUserInvalidJsonMock(t *testing.T) { req, _ := http.NewRequest(http.MethodPost, userPath, bytes.NewBuffer([]byte("invalid json"))) rr := executeRequest(req) diff --git a/httpd/internal_test.go b/httpd/internal_test.go index 06449cc8..0370a5d9 100644 --- a/httpd/internal_test.go +++ b/httpd/internal_test.go @@ -545,6 +545,10 @@ func TestApiCallsWithBadURL(t *testing.T) { assert.Error(t, err) _, _, err = Loaddata("/tmp/backup.json", "", "", http.StatusBadRequest) assert.Error(t, err) + _, _, err = GetBanTime("", http.StatusBadRequest) + assert.Error(t, err) + _, _, err = GetScore("", http.StatusBadRequest) + assert.Error(t, err) SetBaseURLAndCredentials(oldBaseURL, oldAuthUsername, oldAuthPassword) } @@ -597,6 +601,13 @@ func TestApiCallToNotListeningServer(t *testing.T) { assert.Error(t, err) _, _, err = Loaddata("/tmp/backup.json", "", "", http.StatusOK) assert.Error(t, err) + _, _, err = GetBanTime("", http.StatusBadRequest) + assert.Error(t, err) + _, _, err = GetScore("", http.StatusBadRequest) + assert.Error(t, err) + err = UnbanIP("", http.StatusBadRequest) + assert.Error(t, err) + SetBaseURLAndCredentials(oldBaseURL, oldAuthUsername, oldAuthPassword) } diff --git a/httpd/router.go b/httpd/router.go index c62c2313..ff17255a 100644 --- a/httpd/router.go +++ b/httpd/router.go @@ -82,6 +82,9 @@ func initializeRouter(staticFilesPath string, enableWebAdmin bool) { router.Get(loadDataPath, loadData) router.Put(updateUsedQuotaPath, updateUserQuotaUsage) router.Put(updateFolderUsedQuotaPath, updateVFolderQuotaUsage) + router.Get(defenderBanTime, getBanTime) + router.Get(defenderScore, getScore) + router.Post(defenderUnban, unban) if enableWebAdmin { router.Get(webUsersPath, handleGetWebUsers) router.Get(webUserPath, handleWebAddUserGet) diff --git a/httpd/schema/openapi.yaml b/httpd/schema/openapi.yaml index e0c7afa2..7ddd8cf6 100644 --- a/httpd/schema/openapi.yaml +++ b/httpd/schema/openapi.yaml @@ -2,7 +2,7 @@ openapi: 3.0.3 info: title: SFTPGo description: SFTPGo REST API - version: 2.2.4 + version: 2.3.0 servers: - url: /api/v1 @@ -102,6 +102,101 @@ paths: $ref: '#/components/responses/InternalServerError' default: $ref: '#/components/responses/DefaultResponse' + /defender/ban_time: + get: + tags: + - defender + summary: Returns the ban time for the specified IPv4/IPv6 address + operationId: get_ban_time + parameters: + - in: query + name: ip + required: true + description: IPv4/IPv6 address + schema: + type: string + responses: + 200: + description: successful operation + content: + application/json: + schema: + $ref: '#/components/schemas/BanStatus' + 401: + $ref: '#/components/responses/Unauthorized' + 403: + $ref: '#/components/responses/Forbidden' + 404: + $ref: '#/components/responses/NotFound' + 500: + $ref: '#/components/responses/InternalServerError' + default: + $ref: '#/components/responses/DefaultResponse' + /defender/unban: + post: + tags: + - defender + summary: Removes the specified IPv6/IPv6 from the banned ones + operationId: unban_host + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + ip: + type: string + description: IPv4/IPv6 address to remove + responses: + 200: + description: successful operation + content: + application/json: + schema: + $ref: '#/components/schemas/ApiResponse' + 400: + $ref: '#/components/responses/BadRequest' + 401: + $ref: '#/components/responses/Unauthorized' + 403: + $ref: '#/components/responses/Forbidden' + 404: + $ref: '#/components/responses/NotFound' + 500: + $ref: '#/components/responses/InternalServerError' + default: + $ref: '#/components/responses/DefaultResponse' + /defender/score: + get: + tags: + - defender + summary: Returns the score for the specified IPv4/IPv6 address + operationId: get_score + parameters: + - in: query + name: ip + required: true + description: IPv4/IPv6 address + schema: + type: string + responses: + 200: + description: successful operation + content: + application/json: + schema: + $ref: '#/components/schemas/ScoreStatus' + 401: + $ref: '#/components/responses/Unauthorized' + 403: + $ref: '#/components/responses/Forbidden' + 404: + $ref: '#/components/responses/NotFound' + 500: + $ref: '#/components/responses/InternalServerError' + default: + $ref: '#/components/responses/DefaultResponse' /quota_scan: get: tags: @@ -1488,6 +1583,25 @@ components: $ref: '#/components/schemas/WebDAVServiceStatus' data_provider: $ref: '#/components/schemas/DataProviderStatus' + defender: + type: object + properties: + is_active: + type: boolean + BanStatus: + type: object + properties: + date_time: + type: string + format: date-time + nullable: true + description: if null the host is not banned + ScoreStatus: + type: object + properties: + score: + type: integer + description: if 0 the host is not listed ApiResponse: type: object properties: diff --git a/templates/status.html b/templates/status.html index a471b6f2..a5367393 100644 --- a/templates/status.html +++ b/templates/status.html @@ -74,6 +74,15 @@ +
+
+
Defender
+

+ Status: {{ if .Status.Defender.IsActive}}"Enabled"{{else}}"Disabled"{{end}} +

+
+
+
Data provider