add REST API for the defender

common/common.go

@@ -134,6 +134,34 @@ func IsBanned(ip string) bool {
 	return Config.defender.IsBanned(ip)
 }
 
+// GetDefenderBanTime returns the ban time for the given IP
+// or nil if the IP is not banned or the defender is disabled
+func GetDefenderBanTime(ip string) *time.Time {
+	if Config.defender == nil {
+		return nil
+	}
+
+	return Config.defender.GetBanTime(ip)
+}
+
+// Unban removes the specified IP address from the banned ones
+func Unban(ip string) bool {
+	if Config.defender == nil {
+		return false
+	}
+
+	return Config.defender.Unban(ip)
+}
+
+// GetDefenderScore returns the score for the given IP
+func GetDefenderScore(ip string) int {
+	if Config.defender == nil {
+		return 0
+	}
+
+	return Config.defender.GetScore(ip)
+}
+
 // AddDefenderEvent adds the specified defender event for the given IP
 func AddDefenderEvent(ip string, event HostEvent) {
 	if Config.defender == nil {

common/common_test.go

@@ -238,6 +238,10 @@ func TestDefenderIntegration(t *testing.T) {
 	AddDefenderEvent(ip, HostEventNoLoginTried)
 	assert.False(t, IsBanned(ip))
 
+	assert.Nil(t, GetDefenderBanTime(ip))
+	assert.False(t, Unban(ip))
+	assert.Equal(t, 0, GetDefenderScore(ip))
+
 	Config.DefenderConfig = DefenderConfig{
 		Enabled:          true,
 		BanTime:          10,
@@ -257,8 +261,17 @@ func TestDefenderIntegration(t *testing.T) {
 
 	AddDefenderEvent(ip, HostEventNoLoginTried)
 	assert.False(t, IsBanned(ip))
+	assert.Equal(t, 2, GetDefenderScore(ip))
+	assert.False(t, Unban(ip))
+	assert.Nil(t, GetDefenderBanTime(ip))
+
 	AddDefenderEvent(ip, HostEventLoginFailed)
 	assert.True(t, IsBanned(ip))
+	assert.Equal(t, 0, GetDefenderScore(ip))
+	assert.NotNil(t, GetDefenderBanTime(ip))
+	assert.True(t, Unban(ip))
+	assert.Nil(t, GetDefenderBanTime(ip))
+	assert.False(t, Unban(ip))
 
 	Config = configCopy
 }

common/defender.go

@@ -32,6 +32,7 @@ type Defender interface {
 	IsBanned(ip string) bool
 	GetBanTime(ip string) *time.Time
 	GetScore(ip string) int
+	Unban(ip string) bool
 }
 
 // DefenderConfig defines the "defender" configuration
@@ -201,6 +202,19 @@ func (d *memoryDefender) IsBanned(ip string) bool {
 	return false
 }
 
+// Unban removes the specified IP address from the banned ones
+func (d *memoryDefender) Unban(ip string) bool {
+	d.Lock()
+	defer d.Unlock()
+
+	if _, ok := d.banned[ip]; ok {
+		delete(d.banned, ip)
+		return true
+	}
+
+	return false
+}
+
 // AddEvent adds an event for the given IP.
 // This method must be called for clients not yet banned
 func (d *memoryDefender) AddEvent(ip string, event HostEvent) {

common/defender_test.go

@@ -142,6 +142,9 @@ func TestBasicDefender(t *testing.T) {
 		assert.True(t, newBanTime.After(*banTime))
 	}
 
+	assert.True(t, defender.Unban(testIP3))
+	assert.False(t, defender.Unban(testIP3))
+
 	err = os.Remove(slFile)
 	assert.NoError(t, err)
 	err = os.Remove(blFile)

docs/defender.md

@@ -17,13 +17,23 @@ And then you can configure:
 
 So a host is banned, for `ban_time` minutes, if it has exceeded the defined threshold during the last observation time minutes.
 
+A banned IP has no score, it makes no sense to accumulate host events in memory for an already banned IP address.
+
 If an already banned client tries to log in again its ban time will be incremented based on the `ban_time_increment` configuration.
 
 The `ban_time_increment` is calculated as percentage of `ban_time`, so if `ban_time` is 30 minutes and `ban_time_increment` is 50 the host will be banned for additionally 15 minutes. You can specify values greater than 100 for `ban_time_increment`.
 
 The `defender` will keep in memory both the host scores and the banned hosts, you can limit the memory usage using the `entries_soft_limit` and `entries_hard_limit` configuration keys.
 
-The `defender` can also load a permanent block and/or safe list of ip addresses/networks from a file:
+The REST API allows:
+
+- to retrieve the score for an IP address
+- to retrieve the ban time for an IP address
+- to unban an IP address
+
+We don't return the whole list of the banned IP addresses or all the stored scores because we store them as hash map and iterating over all the keys for an hash map is slow and will slow down new events registration.
+
+The `defender` can also load a permanent block list and/or a safe list of ip addresses/networks from a file:
 
 - `safelist_file`, string. Path to a file with a list of ip addresses and/or networks to never ban.
 - `blocklist_file`, string. Path to a file with a list of ip addresses and/or networks to always ban.
@@ -48,6 +58,6 @@ Here is a small example:
 }
 ```
 
-These list will be loaded in memory for faster lookups.
+These list will be loaded in memory for faster lookups. The REST API queries "live" data and not these lists.
 
 The `defender` is optimized for fast and time constant lookups however as it keeps all the lists and the entries in memory you should carefully measure the memory requirements for your use case.

httpd/api_defender.go

@@ -0,0 +1,82 @@
+package httpd
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/go-chi/render"
+
+	"github.com/drakkan/sftpgo/common"
+)
+
+func getBanTime(w http.ResponseWriter, r *http.Request) {
+	ip := r.URL.Query().Get("ip")
+	err := validateIPAddress(ip)
+	if err != nil {
+		sendAPIResponse(w, r, err, "", http.StatusBadRequest)
+		return
+	}
+
+	banStatus := make(map[string]*string)
+
+	banTime := common.GetDefenderBanTime(ip)
+	var banTimeString *string
+	if banTime != nil {
+		rfc3339String := banTime.UTC().Format(time.RFC3339)
+		banTimeString = &rfc3339String
+	}
+
+	banStatus["date_time"] = banTimeString
+	render.JSON(w, r, banStatus)
+}
+
+func getScore(w http.ResponseWriter, r *http.Request) {
+	ip := r.URL.Query().Get("ip")
+	err := validateIPAddress(ip)
+	if err != nil {
+		sendAPIResponse(w, r, err, "", http.StatusBadRequest)
+		return
+	}
+
+	scoreStatus := make(map[string]int)
+	scoreStatus["score"] = common.GetDefenderScore(ip)
+
+	render.JSON(w, r, scoreStatus)
+}
+
+func unban(w http.ResponseWriter, r *http.Request) {
+	r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize)
+
+	var postBody map[string]string
+	err := render.DecodeJSON(r.Body, &postBody)
+	if err != nil {
+		sendAPIResponse(w, r, err, "", http.StatusBadRequest)
+		return
+	}
+
+	ip := postBody["ip"]
+	err = validateIPAddress(ip)
+	if err != nil {
+		sendAPIResponse(w, r, err, "", http.StatusBadRequest)
+		return
+	}
+
+	if common.Unban(ip) {
+		sendAPIResponse(w, r, nil, "OK", http.StatusOK)
+	} else {
+		sendAPIResponse(w, r, nil, "Not found", http.StatusNotFound)
+	}
+}
+
+func validateIPAddress(ip string) error {
+	if ip == "" {
+		return errors.New("ip address is required")
+	}
+	if net.ParseIP(ip) == nil {
+		return fmt.Errorf("ip address %#v is not valid", ip)
+	}
+	return nil
+}

httpd/api_utils.go

@@ -446,6 +446,69 @@ func GetStatus(expectedStatusCode int) (ServicesStatus, []byte, error) {
 	return response, body, err
 }
 
+// GetBanTime returns the ban time for the given IP address
+func GetBanTime(ip string, expectedStatusCode int) (map[string]interface{}, []byte, error) {
+	var response map[string]interface{}
+	var body []byte
+	url, err := url.Parse(buildURLRelativeToBase(defenderBanTime))
+	if err != nil {
+		return response, body, err
+	}
+	q := url.Query()
+	q.Add("ip", ip)
+	url.RawQuery = q.Encode()
+	resp, err := sendHTTPRequest(http.MethodGet, url.String(), nil, "")
+	if err != nil {
+		return response, body, err
+	}
+	defer resp.Body.Close()
+	err = checkResponse(resp.StatusCode, expectedStatusCode)
+	if err == nil && expectedStatusCode == http.StatusOK {
+		err = render.DecodeJSON(resp.Body, &response)
+	} else {
+		body, _ = getResponseBody(resp)
+	}
+	return response, body, err
+}
+
+// GetScore returns the score for the given IP address
+func GetScore(ip string, expectedStatusCode int) (map[string]interface{}, []byte, error) {
+	var response map[string]interface{}
+	var body []byte
+	url, err := url.Parse(buildURLRelativeToBase(defenderScore))
+	if err != nil {
+		return response, body, err
+	}
+	q := url.Query()
+	q.Add("ip", ip)
+	url.RawQuery = q.Encode()
+	resp, err := sendHTTPRequest(http.MethodGet, url.String(), nil, "")
+	if err != nil {
+		return response, body, err
+	}
+	defer resp.Body.Close()
+	err = checkResponse(resp.StatusCode, expectedStatusCode)
+	if err == nil && expectedStatusCode == http.StatusOK {
+		err = render.DecodeJSON(resp.Body, &response)
+	} else {
+		body, _ = getResponseBody(resp)
+	}
+	return response, body, err
+}
+
+// UnbanIP unbans the given IP address
+func UnbanIP(ip string, expectedStatusCode int) error {
+	postBody := make(map[string]string)
+	postBody["ip"] = ip
+	asJSON, _ := json.Marshal(postBody)
+	resp, err := sendHTTPRequest(http.MethodPost, buildURLRelativeToBase(defenderUnban), bytes.NewBuffer(asJSON), "")
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	return checkResponse(resp.StatusCode, expectedStatusCode)
+}
+
 // Dumpdata requests a backup to outputFile.
 // outputFile is relative to the configured backups_path
 func Dumpdata(outputFile, indent string, expectedStatusCode int) (map[string]interface{}, []byte, error) {

httpd/httpd.go

@@ -39,6 +39,9 @@ const (
 	loadDataPath              = "/api/v1/loaddata"
 	updateUsedQuotaPath       = "/api/v1/quota_update"
 	updateFolderUsedQuotaPath = "/api/v1/folder_quota_update"
+	defenderBanTime           = "/api/v1/defender/ban_time"
+	defenderUnban             = "/api/v1/defender/unban"
+	defenderScore             = "/api/v1/defender/score"
 	metricsPath               = "/metrics"
 	webBasePath               = "/web"
 	webUsersPath              = "/web/users"
@@ -61,12 +64,17 @@ var (
 	certMgr     *common.CertManager
 )
 
+type defenderStatus struct {
+	IsActive bool `json:"is_active"`
+}
+
 // ServicesStatus keep the state of the running services
 type ServicesStatus struct {
 	SSH          sftpd.ServiceStatus         `json:"ssh"`
 	FTP          ftpd.ServiceStatus          `json:"ftp"`
 	WebDAV       webdavd.ServiceStatus       `json:"webdav"`
 	DataProvider dataprovider.ProviderStatus `json:"data_provider"`
+	Defender     defenderStatus              `json:"defender"`
 }
 
 // Conf httpd daemon configuration
@@ -186,6 +194,9 @@ func getServicesStatus() ServicesStatus {
 		FTP:          ftpd.GetStatus(),
 		WebDAV:       webdavd.GetStatus(),
 		DataProvider: dataprovider.GetProviderStatus(),
+		Defender: defenderStatus{
+			IsActive: common.Config.DefenderConfig.Enabled,
+		},
 	}
 	return status
 }

httpd/httpd_test.go

@@ -50,6 +50,7 @@ const (
 	quotaScanVFolderPath      = "/api/v1/folder_quota_scan"
 	updateUsedQuotaPath       = "/api/v1/quota_update"
 	updateFolderUsedQuotaPath = "/api/v1/folder_quota_update"
+	defenderUnban             = "/api/v1/defender/unban"
 	versionPath               = "/api/v1/version"
 	metricsPath               = "/metrics"
 	webBasePath               = "/web"
@@ -2200,6 +2201,71 @@ func TestDumpdata(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+func TestDefenderAPI(t *testing.T) {
+	oldConfig := config.GetCommonConfig()
+
+	cfg := config.GetCommonConfig()
+	cfg.DefenderConfig.Enabled = true
+	cfg.DefenderConfig.Threshold = 3
+
+	err := common.Initialize(cfg)
+	require.NoError(t, err)
+
+	ip := "::1"
+
+	response, _, err := httpd.GetBanTime(ip, http.StatusOK)
+	require.NoError(t, err)
+	banTime, ok := response["date_time"]
+	require.True(t, ok)
+	assert.Nil(t, banTime)
+
+	response, _, err = httpd.GetScore(ip, http.StatusOK)
+	require.NoError(t, err)
+	score, ok := response["score"]
+	require.True(t, ok)
+	assert.Equal(t, float64(0), score)
+
+	err = httpd.UnbanIP(ip, http.StatusNotFound)
+	require.NoError(t, err)
+
+	common.AddDefenderEvent(ip, common.HostEventNoLoginTried)
+	response, _, err = httpd.GetScore(ip, http.StatusOK)
+	require.NoError(t, err)
+	score, ok = response["score"]
+	require.True(t, ok)
+	assert.Equal(t, float64(2), score)
+
+	common.AddDefenderEvent(ip, common.HostEventNoLoginTried)
+	response, _, err = httpd.GetBanTime(ip, http.StatusOK)
+	require.NoError(t, err)
+	banTime, ok = response["date_time"]
+	require.True(t, ok)
+	assert.NotNil(t, banTime)
+
+	err = httpd.UnbanIP(ip, http.StatusOK)
+	require.NoError(t, err)
+
+	err = httpd.UnbanIP(ip, http.StatusNotFound)
+	require.NoError(t, err)
+
+	err = common.Initialize(oldConfig)
+	require.NoError(t, err)
+}
+
+func TestDefenderAPIErrors(t *testing.T) {
+	_, _, err := httpd.GetBanTime("", http.StatusBadRequest)
+	require.NoError(t, err)
+
+	_, _, err = httpd.GetBanTime("invalid", http.StatusBadRequest)
+	require.NoError(t, err)
+
+	_, _, err = httpd.GetScore("", http.StatusBadRequest)
+	require.NoError(t, err)
+
+	err = httpd.UnbanIP("", http.StatusBadRequest)
+	require.NoError(t, err)
+}
+
 func TestLoaddata(t *testing.T) {
 	mappedPath := filepath.Join(os.TempDir(), "restored_folder")
 	user := getTestUser()
@@ -2425,6 +2491,12 @@ func TestAddFolderInvalidJsonMock(t *testing.T) {
 	checkResponseCode(t, http.StatusBadRequest, rr.Code)
 }
 
+func TestUnbanInvalidJsonMock(t *testing.T) {
+	req, _ := http.NewRequest(http.MethodPost, defenderUnban, bytes.NewBuffer([]byte("invalid json")))
+	rr := executeRequest(req)
+	checkResponseCode(t, http.StatusBadRequest, rr.Code)
+}
+
 func TestAddUserInvalidJsonMock(t *testing.T) {
 	req, _ := http.NewRequest(http.MethodPost, userPath, bytes.NewBuffer([]byte("invalid json")))
 	rr := executeRequest(req)

httpd/internal_test.go

@@ -545,6 +545,10 @@ func TestApiCallsWithBadURL(t *testing.T) {
 	assert.Error(t, err)
 	_, _, err = Loaddata("/tmp/backup.json", "", "", http.StatusBadRequest)
 	assert.Error(t, err)
+	_, _, err = GetBanTime("", http.StatusBadRequest)
+	assert.Error(t, err)
+	_, _, err = GetScore("", http.StatusBadRequest)
+	assert.Error(t, err)
 	SetBaseURLAndCredentials(oldBaseURL, oldAuthUsername, oldAuthPassword)
 }
 
@@ -597,6 +601,13 @@ func TestApiCallToNotListeningServer(t *testing.T) {
 	assert.Error(t, err)
 	_, _, err = Loaddata("/tmp/backup.json", "", "", http.StatusOK)
 	assert.Error(t, err)
+	_, _, err = GetBanTime("", http.StatusBadRequest)
+	assert.Error(t, err)
+	_, _, err = GetScore("", http.StatusBadRequest)
+	assert.Error(t, err)
+	err = UnbanIP("", http.StatusBadRequest)
+	assert.Error(t, err)
+
 	SetBaseURLAndCredentials(oldBaseURL, oldAuthUsername, oldAuthPassword)
 }
 

httpd/router.go

@@ -82,6 +82,9 @@ func initializeRouter(staticFilesPath string, enableWebAdmin bool) {
 			router.Get(loadDataPath, loadData)
 			router.Put(updateUsedQuotaPath, updateUserQuotaUsage)
 			router.Put(updateFolderUsedQuotaPath, updateVFolderQuotaUsage)
+			router.Get(defenderBanTime, getBanTime)
+			router.Get(defenderScore, getScore)
+			router.Post(defenderUnban, unban)
 			if enableWebAdmin {
 				router.Get(webUsersPath, handleGetWebUsers)
 				router.Get(webUserPath, handleWebAddUserGet)

httpd/schema/openapi.yaml

@@ -2,7 +2,7 @@ openapi: 3.0.3
 info:
   title: SFTPGo
   description: SFTPGo REST API
-  version: 2.2.4
+  version: 2.3.0
 
 servers:
   - url: /api/v1
@@ -102,6 +102,101 @@ paths:
           $ref: '#/components/responses/InternalServerError'
         default:
           $ref: '#/components/responses/DefaultResponse'
+  /defender/ban_time:
+    get:
+      tags:
+        - defender
+      summary: Returns the ban time for the specified IPv4/IPv6 address
+      operationId: get_ban_time
+      parameters:
+        - in: query
+          name: ip
+          required: true
+          description: IPv4/IPv6 address
+          schema:
+            type: string
+      responses:
+        200:
+          description: successful operation
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/BanStatus'
+        401:
+          $ref: '#/components/responses/Unauthorized'
+        403:
+          $ref: '#/components/responses/Forbidden'
+        404:
+          $ref: '#/components/responses/NotFound'
+        500:
+          $ref: '#/components/responses/InternalServerError'
+        default:
+          $ref: '#/components/responses/DefaultResponse'
+  /defender/unban:
+    post:
+      tags:
+        - defender
+      summary: Removes the specified IPv6/IPv6 from the banned ones
+      operationId: unban_host
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                ip:
+                  type: string
+                  description: IPv4/IPv6 address to remove
+      responses:
+        200:
+          description: successful operation
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ApiResponse'
+        400:
+          $ref: '#/components/responses/BadRequest'
+        401:
+          $ref: '#/components/responses/Unauthorized'
+        403:
+          $ref: '#/components/responses/Forbidden'
+        404:
+          $ref: '#/components/responses/NotFound'
+        500:
+          $ref: '#/components/responses/InternalServerError'
+        default:
+          $ref: '#/components/responses/DefaultResponse'
+  /defender/score:
+    get:
+      tags:
+        - defender
+      summary: Returns the score for the specified IPv4/IPv6 address
+      operationId: get_score
+      parameters:
+        - in: query
+          name: ip
+          required: true
+          description: IPv4/IPv6 address
+          schema:
+            type: string
+      responses:
+        200:
+          description: successful operation
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ScoreStatus'
+        401:
+          $ref: '#/components/responses/Unauthorized'
+        403:
+          $ref: '#/components/responses/Forbidden'
+        404:
+          $ref: '#/components/responses/NotFound'
+        500:
+          $ref: '#/components/responses/InternalServerError'
+        default:
+          $ref: '#/components/responses/DefaultResponse'
   /quota_scan:
     get:
       tags:
@@ -1488,6 +1583,25 @@ components:
           $ref: '#/components/schemas/WebDAVServiceStatus'
         data_provider:
           $ref: '#/components/schemas/DataProviderStatus'
+        defender:
+          type: object
+          properties:
+            is_active:
+              type: boolean
+    BanStatus:
+      type: object
+      properties:
+        date_time:
+          type: string
+          format: date-time
+          nullable: true
+          description: if null the host is not banned
+    ScoreStatus:
+      type: object
+      properties:
+        score:
+          type: integer
+          description: if 0 the host is not listed
     ApiResponse:
       type: object
       properties:

templates/status.html

@@ -74,6 +74,15 @@
     </div>
 </div>
 
+<div class="card mb-4 {{ if .Status.Defender.IsActive}}border-left-success{{else}}border-left-info{{end}}">
+    <div class="card-body">
+        <h5 class="card-title">Defender</h5>
+        <p class="card-text">
+            Status: {{ if .Status.Defender.IsActive}}"Enabled"{{else}}"Disabled"{{end}}
+        </p>
+    </div>
+</div>
+
 <div class="card mb-4 {{ if .Status.DataProvider.IsActive}}border-left-success{{else}}border-left-warning{{end}}">
     <div class="card-body">
         <h5 class="card-title">Data provider</h5>