Fix #1552 (#1569)

cmd/crowdsec-cli/decisions.go

@@ -284,7 +284,7 @@ cscli decisions list -t ban
 	cmdDecisionsList.Flags().StringVar(filter.Until, "until", "", "restrict to alerts older than until (ie. 4h, 30d)")
 	cmdDecisionsList.Flags().StringVarP(filter.TypeEquals, "type", "t", "", "restrict to this decision type (ie. ban,captcha)")
 	cmdDecisionsList.Flags().StringVar(filter.ScopeEquals, "scope", "", "restrict to this scope (ie. ip,range,session)")
-	cmdDecisionsList.Flags().StringVar(filter.OriginEquals, "origin", "", "restrict to this origin (ie. lists,CAPI,cscli)")
+	cmdDecisionsList.Flags().StringVar(filter.OriginEquals, "origin", "", "restrict to this origin (ie. lists,CAPI,cscli,cscli-import,crowdsec)")
 	cmdDecisionsList.Flags().StringVarP(filter.ValueEquals, "value", "v", "", "restrict to this value (ie. 1.2.3.4,userName)")
 	cmdDecisionsList.Flags().StringVarP(filter.ScenarioEquals, "scenario", "s", "", "restrict to this scenario (ie. crowdsecurity/ssh-bf)")
 	cmdDecisionsList.Flags().StringVarP(filter.IPEquals, "ip", "i", "", "restrict to alerts from this source ip (shorthand for --scope ip --value <IP>)")
@@ -419,7 +419,6 @@ cscli decisions add --scope username --value foobar
 		Aliases:           []string{"remove"},
 		Example: `cscli decisions delete -r 1.2.3.0/24
 cscli decisions delete -i 1.2.3.4
-cscli decisions delete -s crowdsecurity/ssh-bf
 cscli decisions delete --id 42
 cscli decisions delete --type captcha
 `,

pkg/apiserver/alerts_test.go

@@ -23,6 +23,7 @@ type LAPI struct {
 	loginResp  models.WatcherAuthResponse
 	bouncerKey string
 	t          *testing.T
+	DBConfig   *csconfig.DatabaseCfg
 }
 
 func SetupLAPITest(t *testing.T) LAPI {
@@ -36,10 +37,12 @@ func SetupLAPITest(t *testing.T) LAPI {
 	if err != nil {
 		t.Fatalf("%s", err.Error())
 	}
+
 	return LAPI{
 		router:     router,
 		loginResp:  loginResp,
 		bouncerKey: APIKey,
+		DBConfig:   config.API.Server.DbConfig,
 	}
 }
 

pkg/apiserver/controllers/v1/decisions.go

@@ -11,9 +11,20 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
-func FormatDecisions(decisions []*ent.Decision) ([]*models.Decision, error) {
+//Format decisions for the bouncers, and deduplicate them by keeping only the longest one
+func FormatDecisions(decisions []*ent.Decision, dedup bool) ([]*models.Decision, error) {
 	var results []*models.Decision
+
+	seen := make(map[string]struct{}, 0)
+
 	for _, dbDecision := range decisions {
+		if dedup {
+			key := dbDecision.Value + dbDecision.Scope + dbDecision.Type
+			if _, ok := seen[key]; ok {
+				continue
+			}
+			seen[key] = struct{}{}
+		}
 		duration := dbDecision.Until.Sub(time.Now().UTC()).String()
 		decision := models.Decision{
 			ID:       int64(dbDecision.ID),
@@ -46,7 +57,7 @@ func (c *Controller) GetDecision(gctx *gin.Context) {
 		return
 	}
 
-	results, err = FormatDecisions(data)
+	results, err = FormatDecisions(data, false)
 	if err != nil {
 		gctx.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()})
 		return
@@ -82,14 +93,14 @@ func (c *Controller) DeleteDecisionById(gctx *gin.Context) {
 		gctx.JSON(http.StatusBadRequest, gin.H{"message": "decision_id must be valid integer"})
 		return
 	}
-	err = c.DBClient.SoftDeleteDecisionByID(decisionID)
+	nbDeleted, err := c.DBClient.SoftDeleteDecisionByID(decisionID)
 	if err != nil {
 		c.HandleDBErrors(gctx, err)
 		return
 	}
 
 	deleteDecisionResp := models.DeleteDecisionResponse{
-		NbDeleted: "1",
+		NbDeleted: strconv.Itoa(nbDeleted),
 	}
 
 	gctx.JSON(http.StatusOK, deleteDecisionResp)
@@ -138,7 +149,8 @@ func (c *Controller) StreamDecision(gctx *gin.Context) {
 				gctx.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()})
 				return
 			}
-			ret["new"], err = FormatDecisions(data)
+			//data = KeepLongestDecision(data)
+			ret["new"], err = FormatDecisions(data, true)
 			if err != nil {
 				log.Errorf("unable to format expired decision for '%s' : %v", bouncerInfo.Name, err)
 				gctx.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()})
@@ -152,7 +164,7 @@ func (c *Controller) StreamDecision(gctx *gin.Context) {
 				gctx.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()})
 				return
 			}
-			ret["deleted"], err = FormatDecisions(data)
+			ret["deleted"], err = FormatDecisions(data, true)
 			if err != nil {
 				log.Errorf("unable to format expired decision for '%s' : %v", bouncerInfo.Name, err)
 				gctx.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()})
@@ -180,7 +192,8 @@ func (c *Controller) StreamDecision(gctx *gin.Context) {
 		gctx.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()})
 		return
 	}
-	ret["new"], err = FormatDecisions(data)
+	//data = KeepLongestDecision(data)
+	ret["new"], err = FormatDecisions(data, true)
 	if err != nil {
 		log.Errorf("unable to format new decision for '%s' : %v", bouncerInfo.Name, err)
 		gctx.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()})
@@ -194,7 +207,7 @@ func (c *Controller) StreamDecision(gctx *gin.Context) {
 		gctx.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()})
 		return
 	}
-	ret["deleted"], err = FormatDecisions(data)
+	ret["deleted"], err = FormatDecisions(data, true)
 	if err != nil {
 		log.Errorf("unable to format expired decision for '%s' : %v", bouncerInfo.Name, err)
 		gctx.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()})

pkg/apiserver/decisions_test.go

@@ -1,12 +1,18 @@
 package apiserver
 
 import (
+	"fmt"
+	"os"
 	"testing"
-	"time"
 
 	"github.com/stretchr/testify/assert"
 )
 
+const (
+	APIKEY   = "apikey"
+	PASSWORD = "password"
+)
+
 func TestDeleteDecisionRange(t *testing.T) {
 	lapi := SetupLAPITest(t)
 
@@ -14,20 +20,20 @@ func TestDeleteDecisionRange(t *testing.T) {
 	lapi.InsertAlertFromFile("./tests/alert_minibulk.json")
 
 	// delete by ip wrong
-	w := lapi.RecordResponse("DELETE", "/v1/decisions?range=1.2.3.0/24", emptyBody, "password")
+	w := lapi.RecordResponse("DELETE", "/v1/decisions?range=1.2.3.0/24", emptyBody, PASSWORD)
 	assert.Equal(t, 200, w.Code)
 
 	assert.Equal(t, `{"nbDeleted":"0"}`, w.Body.String())
 
 	// delete by range
 
-	w = lapi.RecordResponse("DELETE", "/v1/decisions?range=91.121.79.0/24&contains=false", emptyBody, "password")
+	w = lapi.RecordResponse("DELETE", "/v1/decisions?range=91.121.79.0/24&contains=false", emptyBody, PASSWORD)
 	assert.Equal(t, 200, w.Code)
 	assert.Equal(t, `{"nbDeleted":"2"}`, w.Body.String())
 
 	// delete by range : ensure it was already deleted
 
-	w = lapi.RecordResponse("DELETE", "/v1/decisions?range=91.121.79.0/24", emptyBody, "password")
+	w = lapi.RecordResponse("DELETE", "/v1/decisions?range=91.121.79.0/24", emptyBody, PASSWORD)
 	assert.Equal(t, 200, w.Code)
 	assert.Equal(t, `{"nbDeleted":"0"}`, w.Body.String())
 }
@@ -40,19 +46,19 @@ func TestDeleteDecisionFilter(t *testing.T) {
 
 	// delete by ip wrong
 
-	w := lapi.RecordResponse("DELETE", "/v1/decisions?ip=1.2.3.4", emptyBody, "password")
+	w := lapi.RecordResponse("DELETE", "/v1/decisions?ip=1.2.3.4", emptyBody, PASSWORD)
 	assert.Equal(t, 200, w.Code)
 	assert.Equal(t, `{"nbDeleted":"0"}`, w.Body.String())
 
 	// delete by ip good
 
-	w = lapi.RecordResponse("DELETE", "/v1/decisions?ip=91.121.79.179", emptyBody, "password")
+	w = lapi.RecordResponse("DELETE", "/v1/decisions?ip=91.121.79.179", emptyBody, PASSWORD)
 	assert.Equal(t, 200, w.Code)
 	assert.Equal(t, `{"nbDeleted":"1"}`, w.Body.String())
 
 	// delete by scope/value
 
-	w = lapi.RecordResponse("DELETE", "/v1/decisions?scopes=Ip&value=91.121.79.178", emptyBody, "password")
+	w = lapi.RecordResponse("DELETE", "/v1/decisions?scopes=Ip&value=91.121.79.178", emptyBody, PASSWORD)
 	assert.Equal(t, 200, w.Code)
 	assert.Equal(t, `{"nbDeleted":"1"}`, w.Body.String())
 }
@@ -65,7 +71,7 @@ func TestGetDecisionFilters(t *testing.T) {
 
 	// Get Decision
 
-	w := lapi.RecordResponse("GET", "/v1/decisions", emptyBody, "apikey")
+	w := lapi.RecordResponse("GET", "/v1/decisions", emptyBody, APIKEY)
 	assert.Equal(t, 200, w.Code)
 	decisions, code, err := readDecisionsGetResp(w)
 	assert.Nil(t, err)
@@ -80,7 +86,7 @@ func TestGetDecisionFilters(t *testing.T) {
 
 	// Get Decision : type filter
 
-	w = lapi.RecordResponse("GET", "/v1/decisions?type=ban", emptyBody, "apikey")
+	w = lapi.RecordResponse("GET", "/v1/decisions?type=ban", emptyBody, APIKEY)
 	assert.Equal(t, 200, w.Code)
 	decisions, code, err = readDecisionsGetResp(w)
 	assert.Nil(t, err)
@@ -98,7 +104,7 @@ func TestGetDecisionFilters(t *testing.T) {
 
 	// Get Decision : scope/value
 
-	w = lapi.RecordResponse("GET", "/v1/decisions?scopes=Ip&value=91.121.79.179", emptyBody, "apikey")
+	w = lapi.RecordResponse("GET", "/v1/decisions?scopes=Ip&value=91.121.79.179", emptyBody, APIKEY)
 	assert.Equal(t, 200, w.Code)
 	decisions, code, err = readDecisionsGetResp(w)
 	assert.Nil(t, err)
@@ -113,7 +119,7 @@ func TestGetDecisionFilters(t *testing.T) {
 
 	// Get Decision : ip filter
 
-	w = lapi.RecordResponse("GET", "/v1/decisions?ip=91.121.79.179", emptyBody, "apikey")
+	w = lapi.RecordResponse("GET", "/v1/decisions?ip=91.121.79.179", emptyBody, APIKEY)
 	assert.Equal(t, 200, w.Code)
 	decisions, code, err = readDecisionsGetResp(w)
 	assert.Nil(t, err)
@@ -127,7 +133,7 @@ func TestGetDecisionFilters(t *testing.T) {
 	// assert.NotContains(t, w.Body.String(), `"id":2,"origin":"crowdsec","scenario":"crowdsecurity/ssh-bf","scope":"Ip","type":"ban","value":"91.121.79.178"`)
 
 	// Get decision : by range
-	w = lapi.RecordResponse("GET", "/v1/decisions?range=91.121.79.0/24&contains=false", emptyBody, "apikey")
+	w = lapi.RecordResponse("GET", "/v1/decisions?range=91.121.79.0/24&contains=false", emptyBody, APIKEY)
 	assert.Equal(t, 200, w.Code)
 	decisions, code, err = readDecisionsGetResp(w)
 	assert.Nil(t, err)
@@ -145,7 +151,7 @@ func TestGetDecision(t *testing.T) {
 	lapi.InsertAlertFromFile("./tests/alert_sample.json")
 
 	// Get Decision
-	w := lapi.RecordResponse("GET", "/v1/decisions", emptyBody, "apikey")
+	w := lapi.RecordResponse("GET", "/v1/decisions", emptyBody, APIKEY)
 	assert.Equal(t, 200, w.Code)
 	decisions, code, err := readDecisionsGetResp(w)
 	assert.Nil(t, err)
@@ -165,7 +171,7 @@ func TestGetDecision(t *testing.T) {
 	assert.Equal(t, int64(3), decisions[2].ID)
 
 	// Get Decision with invalid filter. It should ignore this filter
-	w = lapi.RecordResponse("GET", "/v1/decisions?test=test", emptyBody, "apikey")
+	w = lapi.RecordResponse("GET", "/v1/decisions?test=test", emptyBody, APIKEY)
 	assert.Equal(t, 200, w.Code)
 	assert.Equal(t, 3, len(decisions))
 }
@@ -177,49 +183,49 @@ func TestDeleteDecisionByID(t *testing.T) {
 	lapi.InsertAlertFromFile("./tests/alert_sample.json")
 
 	//Have one alerts
-	w := lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody, "apikey")
+	w := lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody, APIKEY)
 	decisions, code, err := readDecisionsStreamResp(w)
 	assert.Equal(t, err, nil)
-	assert.Equal(t, code, 200)
-	assert.Equal(t, len(decisions["deleted"]), 0)
-	assert.Equal(t, len(decisions["new"]), 1)
+	assert.Equal(t, 200, code)
+	assert.Equal(t, 0, len(decisions["deleted"]))
+	assert.Equal(t, 1, len(decisions["new"]))
 
 	// Delete alert with Invalid ID
-	w = lapi.RecordResponse("DELETE", "/v1/decisions/test", emptyBody, "password")
+	w = lapi.RecordResponse("DELETE", "/v1/decisions/test", emptyBody, PASSWORD)
 	assert.Equal(t, 400, w.Code)
 	err_resp, _, err := readDecisionsErrorResp(w)
 	assert.NoError(t, err)
-	assert.Equal(t, err_resp["message"], "decision_id must be valid integer")
+	assert.Equal(t, "decision_id must be valid integer", err_resp["message"])
 
 	// Delete alert with ID that not exist
-	w = lapi.RecordResponse("DELETE", "/v1/decisions/100", emptyBody, "password")
+	w = lapi.RecordResponse("DELETE", "/v1/decisions/100", emptyBody, PASSWORD)
 	assert.Equal(t, 500, w.Code)
 	err_resp, _, err = readDecisionsErrorResp(w)
 	assert.NoError(t, err)
-	assert.Equal(t, err_resp["message"], "decision with id '100' doesn't exist: unable to delete")
+	assert.Equal(t, "decision with id '100' doesn't exist: unable to delete", err_resp["message"])
 
 	//Have one alerts
-	w = lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody, "apikey")
+	w = lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody, APIKEY)
 	decisions, code, err = readDecisionsStreamResp(w)
 	assert.Equal(t, err, nil)
-	assert.Equal(t, code, 200)
-	assert.Equal(t, len(decisions["deleted"]), 0)
-	assert.Equal(t, len(decisions["new"]), 1)
+	assert.Equal(t, 200, code)
+	assert.Equal(t, 0, len(decisions["deleted"]))
+	assert.Equal(t, 1, len(decisions["new"]))
 
 	// Delete alert with valid ID
-	w = lapi.RecordResponse("DELETE", "/v1/decisions/1", emptyBody, "password")
+	w = lapi.RecordResponse("DELETE", "/v1/decisions/1", emptyBody, PASSWORD)
 	assert.Equal(t, 200, w.Code)
 	resp, _, err := readDecisionsDeleteResp(w)
 	assert.NoError(t, err)
 	assert.Equal(t, resp.NbDeleted, "1")
 
 	//Have one alert (because we delete an alert that has dup targets)
-	w = lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody, "apikey")
+	w = lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody, APIKEY)
 	decisions, code, err = readDecisionsStreamResp(w)
 	assert.Equal(t, err, nil)
-	assert.Equal(t, code, 200)
-	assert.Equal(t, len(decisions["deleted"]), 0)
-	assert.Equal(t, len(decisions["new"]), 1)
+	assert.Equal(t, 200, code)
+	assert.Equal(t, 0, len(decisions["deleted"]))
+	assert.Equal(t, 1, len(decisions["new"]))
 }
 
 func TestDeleteDecision(t *testing.T) {
@@ -229,14 +235,14 @@ func TestDeleteDecision(t *testing.T) {
 	lapi.InsertAlertFromFile("./tests/alert_sample.json")
 
 	// Delete alert with Invalid filter
-	w := lapi.RecordResponse("DELETE", "/v1/decisions?test=test", emptyBody, "password")
+	w := lapi.RecordResponse("DELETE", "/v1/decisions?test=test", emptyBody, PASSWORD)
 	assert.Equal(t, 500, w.Code)
 	err_resp, _, err := readDecisionsErrorResp(w)
 	assert.NoError(t, err)
 	assert.Equal(t, err_resp["message"], "'test' doesn't exist: invalid filter")
 
 	// Delete all alert
-	w = lapi.RecordResponse("DELETE", "/v1/decisions", emptyBody, "password")
+	w = lapi.RecordResponse("DELETE", "/v1/decisions", emptyBody, PASSWORD)
 	assert.Equal(t, 200, w.Code)
 	resp, _, err := readDecisionsDeleteResp(w)
 	assert.NoError(t, err)
@@ -251,181 +257,1061 @@ func TestStreamStartDecisionDedup(t *testing.T) {
 	lapi.InsertAlertFromFile("./tests/alert_sample.json")
 
 	// Get Stream, we only get one decision (the longest one)
-	w := lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody, "apikey")
+	w := lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody, APIKEY)
 	decisions, code, err := readDecisionsStreamResp(w)
-	assert.Equal(t, err, nil)
-	assert.Equal(t, code, 200)
-	assert.Equal(t, len(decisions["deleted"]), 0)
-	assert.Equal(t, len(decisions["new"]), 1)
-	assert.Equal(t, decisions["new"][0].ID, int64(3))
-	assert.Equal(t, *decisions["new"][0].Origin, "test")
-	assert.Equal(t, *decisions["new"][0].Value, "127.0.0.1")
+	assert.Equal(t, nil, err)
+	assert.Equal(t, 200, code)
+	assert.Equal(t, 0, len(decisions["deleted"]))
+	assert.Equal(t, 1, len(decisions["new"]))
+	assert.Equal(t, int64(3), decisions["new"][0].ID)
+	assert.Equal(t, "test", *decisions["new"][0].Origin)
+	assert.Equal(t, "127.0.0.1", *decisions["new"][0].Value)
 
 	// id=3 decision is deleted, this won't affect `deleted`, because there are decisions on the same ip
-	w = lapi.RecordResponse("DELETE", "/v1/decisions/3", emptyBody, "password")
+	w = lapi.RecordResponse("DELETE", "/v1/decisions/3", emptyBody, PASSWORD)
 	assert.Equal(t, 200, w.Code)
 
 	// Get Stream, we only get one decision (the longest one, id=2)
-	w = lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody, "apikey")
+	w = lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody, APIKEY)
 	decisions, code, err = readDecisionsStreamResp(w)
-	assert.Equal(t, err, nil)
-	assert.Equal(t, code, 200)
-	assert.Equal(t, len(decisions["deleted"]), 0)
-	assert.Equal(t, len(decisions["new"]), 1)
-	assert.Equal(t, decisions["new"][0].ID, int64(2))
-	assert.Equal(t, *decisions["new"][0].Origin, "test")
-	assert.Equal(t, *decisions["new"][0].Value, "127.0.0.1")
+	assert.Equal(t, nil, err)
+	assert.Equal(t, 200, code)
+	assert.Equal(t, 0, len(decisions["deleted"]))
+	assert.Equal(t, 1, len(decisions["new"]))
+	assert.Equal(t, int64(2), decisions["new"][0].ID)
+	assert.Equal(t, "test", *decisions["new"][0].Origin)
+	assert.Equal(t, "127.0.0.1", *decisions["new"][0].Value)
 
 	// We delete another decision, yet don't receive it in stream, since there's another decision on same IP
-	w = lapi.RecordResponse("DELETE", "/v1/decisions/2", emptyBody, "password")
+	w = lapi.RecordResponse("DELETE", "/v1/decisions/2", emptyBody, PASSWORD)
 	assert.Equal(t, 200, w.Code)
 
 	// And get the remaining decision (1)
-	w = lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody, "apikey")
+	w = lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody, APIKEY)
 	decisions, code, err = readDecisionsStreamResp(w)
-	assert.Equal(t, err, nil)
-	assert.Equal(t, code, 200)
-	assert.Equal(t, len(decisions["deleted"]), 0)
-	assert.Equal(t, len(decisions["new"]), 1)
-	assert.Equal(t, decisions["new"][0].ID, int64(1))
-	assert.Equal(t, *decisions["new"][0].Origin, "test")
-	assert.Equal(t, *decisions["new"][0].Value, "127.0.0.1")
+	assert.Equal(t, nil, err)
+	assert.Equal(t, 200, code)
+	assert.Equal(t, 0, len(decisions["deleted"]))
+	assert.Equal(t, 1, len(decisions["new"]))
+	assert.Equal(t, int64(1), decisions["new"][0].ID)
+	assert.Equal(t, "test", *decisions["new"][0].Origin)
+	assert.Equal(t, "127.0.0.1", *decisions["new"][0].Value)
 
 	// We delete the last decision, we receive the delete order
-	w = lapi.RecordResponse("DELETE", "/v1/decisions/1", emptyBody, "password")
+	w = lapi.RecordResponse("DELETE", "/v1/decisions/1", emptyBody, PASSWORD)
 	assert.Equal(t, 200, w.Code)
 
 	//and now we only get a deleted decision
-	w = lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody, "apikey")
+	w = lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody, APIKEY)
 	decisions, code, err = readDecisionsStreamResp(w)
-	assert.Equal(t, err, nil)
-	assert.Equal(t, code, 200)
-	assert.Equal(t, len(decisions["deleted"]), 1)
-	assert.Equal(t, decisions["deleted"][0].ID, int64(1))
-	assert.Equal(t, *decisions["deleted"][0].Origin, "test")
-	assert.Equal(t, *decisions["deleted"][0].Value, "127.0.0.1")
-	assert.Equal(t, len(decisions["new"]), 0)
+	assert.Equal(t, nil, err)
+	assert.Equal(t, 200, code)
+	assert.Equal(t, 1, len(decisions["deleted"]))
+	assert.Equal(t, int64(1), decisions["deleted"][0].ID)
+	assert.Equal(t, "test", *decisions["deleted"][0].Origin)
+	assert.Equal(t, "127.0.0.1", *decisions["deleted"][0].Value)
+	assert.Equal(t, 0, len(decisions["new"]))
 }
 
-func TestStreamDecisionDedup(t *testing.T) {
-	//Ensure that at stream startup we only get the longest decision
-	lapi := SetupLAPITest(t)
-
-	// Create Valid Alert : 3 decisions for 127.0.0.1, longest has id=3
-	lapi.InsertAlertFromFile("./tests/alert_sample.json")
-
-	time.Sleep(2 * time.Second)
-
-	// Get Stream, we only get one decision (the longest one)
-	w := lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody, "apikey")
-	decisions, code, err := readDecisionsStreamResp(w)
-	assert.Equal(t, err, nil)
-	assert.Equal(t, code, 200)
-	assert.Equal(t, len(decisions["deleted"]), 0)
-	assert.Equal(t, len(decisions["new"]), 1)
-	assert.Equal(t, decisions["new"][0].ID, int64(3))
-	assert.Equal(t, *decisions["new"][0].Origin, "test")
-	assert.Equal(t, *decisions["new"][0].Value, "127.0.0.1")
-
-	// id=3 decision is deleted, this won't affect `deleted`, because there are decisions on the same ip
-	w = lapi.RecordResponse("DELETE", "/v1/decisions/3", emptyBody, "password")
-	assert.Equal(t, 200, w.Code)
-
-	w = lapi.RecordResponse("GET", "/v1/decisions/stream", emptyBody, "apikey")
-	assert.Equal(t, err, nil)
-	decisions, code, err = readDecisionsStreamResp(w)
-	assert.Equal(t, err, nil)
-	assert.Equal(t, code, 200)
-	assert.Equal(t, len(decisions["deleted"]), 0)
-	assert.Equal(t, len(decisions["new"]), 0)
-	// We delete another decision, yet don't receive it in stream, since there's another decision on same IP
-	w = lapi.RecordResponse("DELETE", "/v1/decisions/2", emptyBody, "password")
-	assert.Equal(t, 200, w.Code)
-
-	w = lapi.RecordResponse("GET", "/v1/decisions/stream", emptyBody, "apikey")
-	decisions, code, err = readDecisionsStreamResp(w)
-	assert.Equal(t, err, nil)
-	assert.Equal(t, code, 200)
-	assert.Equal(t, len(decisions["deleted"]), 0)
-	assert.Equal(t, len(decisions["new"]), 0)
-
-	// We delete the last decision, we receive the delete order
-	w = lapi.RecordResponse("DELETE", "/v1/decisions/1", emptyBody, "password")
-	assert.Equal(t, 200, w.Code)
-
-	w = lapi.RecordResponse("GET", "/v1/decisions/stream", emptyBody, "apikey")
-	decisions, code, err = readDecisionsStreamResp(w)
-	assert.Equal(t, err, nil)
-	assert.Equal(t, code, 200)
-	assert.Equal(t, len(decisions["deleted"]), 1)
-	assert.Equal(t, decisions["deleted"][0].ID, int64(1))
-	assert.Equal(t, *decisions["deleted"][0].Origin, "test")
-	assert.Equal(t, *decisions["deleted"][0].Value, "127.0.0.1")
-	assert.Equal(t, len(decisions["new"]), 0)
+type DecisionCheck struct {
+	ID       int64
+	Origin   string
+	Scenario string
+	Value    string
+	Duration string
+	Type     string
 }
 
-func TestStreamDecisionFilters(t *testing.T) {
+type DecisionTest struct {
+	TestName      string
+	Method        string
+	Route         string
+	CheckCodeOnly bool
+	Code          int
+	LenNew        int
+	LenDeleted    int
+	NewChecks     []DecisionCheck
+	DelChecks     []DecisionCheck
+	AuthType      string
+}
 
+func TestStreamDecisionStart(t *testing.T) {
 	lapi := SetupLAPITest(t)
 
-	// Create Valid Alert
-	lapi.InsertAlertFromFile("./tests/alert_stream_fixture.json")
-
-	w := lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true", emptyBody, "apikey")
-	decisions, code, err := readDecisionsStreamResp(w)
-
-	assert.Equal(t, 200, code)
-	assert.Equal(t, err, nil)
-	assert.Equal(t, len(decisions["deleted"]), 0)
-	assert.Equal(t, len(decisions["new"]), 3)
-	assert.Equal(t, decisions["new"][0].ID, int64(1))
-	assert.Equal(t, *decisions["new"][0].Origin, "test1")
-	assert.Equal(t, *decisions["new"][0].Value, "127.0.0.1")
-	assert.Equal(t, *decisions["new"][0].Scenario, "crowdsecurity/http_bf")
-	assert.Equal(t, decisions["new"][1].ID, int64(2))
-	assert.Equal(t, *decisions["new"][1].Origin, "test2")
-	assert.Equal(t, *decisions["new"][1].Value, "127.0.0.1")
-	assert.Equal(t, *decisions["new"][1].Scenario, "crowdsecurity/ssh_bf")
-	assert.Equal(t, decisions["new"][2].ID, int64(3))
-	assert.Equal(t, *decisions["new"][2].Origin, "test3")
-	assert.Equal(t, *decisions["new"][2].Value, "127.0.0.1")
-	assert.Equal(t, *decisions["new"][2].Scenario, "crowdsecurity/ddos")
-
-	// test filter scenarios_not_containing
-	w = lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true&scenarios_not_containing=http", emptyBody, "apikey")
-	decisions, code, err = readDecisionsStreamResp(w)
-	assert.Equal(t, err, nil)
-	assert.Equal(t, 200, code)
-	assert.Equal(t, len(decisions["deleted"]), 0)
-	assert.Equal(t, len(decisions["new"]), 2)
-	assert.Equal(t, decisions["new"][0].ID, int64(2))
-	assert.Equal(t, decisions["new"][1].ID, int64(3))
-
-	// test  filter scenarios_containing
-	w = lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true&scenarios_containing=http", emptyBody, "apikey")
-	decisions, code, err = readDecisionsStreamResp(w)
-	assert.Equal(t, err, nil)
-	assert.Equal(t, 200, code)
-	assert.Equal(t, len(decisions["deleted"]), 0)
-	assert.Equal(t, len(decisions["new"]), 1)
-	assert.Equal(t, decisions["new"][0].ID, int64(1))
+	/*
+		Create multiple alerts:
+		  - 3 alerts for 127.0.0.1 with ID 1/2/3   : Different duration / scenario / origin
+		  - 3 alerts for 127.0.0.2 with ID 4/5/6/7 : Different duration / scenario / origin
+	*/
+	lapi.InsertAlertFromFile("./tests/alert_duplicate.json")
+
+	tests := []DecisionTest{
+		{
+			TestName:      "test startup",
+			Method:        "GET",
+			Route:         "/v1/decisions/stream?startup=true",
+			CheckCodeOnly: false,
+			Code:          200,
+			LenNew:        3,
+			LenDeleted:    0,
+			AuthType:      APIKEY,
+			DelChecks:     []DecisionCheck{},
+			NewChecks: []DecisionCheck{
+				{
+					ID:       int64(3),
+					Origin:   "test",
+					Scenario: "crowdsecurity/longest",
+					Value:    "127.0.0.1",
+					Duration: "4h59",
+					Type:     "ban",
+				},
+				{
+					ID:       int64(4),
+					Origin:   "test",
+					Scenario: "crowdsecurity/test",
+					Value:    "127.0.0.2",
+					Duration: "2h59",
+					Type:     "ban",
+				},
+				{
+					ID:       int64(8),
+					Origin:   "test",
+					Scenario: "crowdsecurity/test",
+					Value:    "127.0.0.2",
+					Duration: "2h59",
+					Type:     "captcha",
+				},
+			},
+		},
+		{
+			TestName:      "test startup with scenarios containing",
+			Method:        "GET",
+			Route:         "/v1/decisions/stream?startup=true&scenarios_containing=ssh_bf",
+			CheckCodeOnly: false,
+			Code:          200,
+			LenNew:        2,
+			LenDeleted:    0,
+			AuthType:      APIKEY,
+			DelChecks:     []DecisionCheck{},
+			NewChecks: []DecisionCheck{
+				{
+					ID:       int64(2),
+					Origin:   "another_origin",
+					Scenario: "crowdsecurity/ssh_bf",
+					Value:    "127.0.0.1",
+					Duration: "2h59",
+					Type:     "ban",
+				},
+				{
+					ID:       int64(5),
+					Origin:   "test",
+					Scenario: "crowdsecurity/ssh_bf",
+					Value:    "127.0.0.2",
+					Duration: "2h59",
+					Type:     "ban",
+				},
+			},
+		},
+		{
+			TestName:      "test startup with multiple scenarios containing",
+			Method:        "GET",
+			Route:         "/v1/decisions/stream?startup=true&scenarios_containing=ssh_bf,test",
+			CheckCodeOnly: false,
+			Code:          200,
+			LenNew:        3,
+			LenDeleted:    0,
+			AuthType:      APIKEY,
+			DelChecks:     []DecisionCheck{},
+
+			NewChecks: []DecisionCheck{
+				{
+					ID:       int64(2),
+					Origin:   "another_origin",
+					Scenario: "crowdsecurity/ssh_bf",
+					Value:    "127.0.0.1",
+					Duration: "2h59",
+					Type:     "ban",
+				},
+				{
+					ID:       int64(4),
+					Origin:   "test",
+					Scenario: "crowdsecurity/test",
+					Value:    "127.0.0.2",
+					Duration: "2h59",
+					Type:     "ban",
+				},
+				{
+					ID:       int64(8),
+					Origin:   "test",
+					Scenario: "crowdsecurity/test",
+					Value:    "127.0.0.2",
+					Duration: "2h59",
+					Type:     "captcha",
+				},
+			},
+		},
+		{
+			TestName:      "test startup with unknown scenarios containing",
+			Method:        "GET",
+			Route:         "/v1/decisions/stream?startup=true&scenarios_containing=unknown",
+			CheckCodeOnly: false,
+			Code:          200,
+			LenNew:        0,
+			LenDeleted:    0,
+			AuthType:      APIKEY,
+			DelChecks:     []DecisionCheck{},
+
+			NewChecks: []DecisionCheck{},
+		},
+		{
+			TestName:      "test startup with scenarios containing and not containing",
+			Method:        "GET",
+			Route:         "/v1/decisions/stream?startup=true&scenarios_containing=test&scenarios_not_containing=ssh_bf",
+			CheckCodeOnly: false,
+			Code:          200,
+			LenNew:        3,
+			LenDeleted:    0,
+			AuthType:      APIKEY,
+			DelChecks:     []DecisionCheck{},
+			NewChecks: []DecisionCheck{
+				{
+					ID:       int64(1),
+					Origin:   "test",
+					Scenario: "crowdsecurity/test",
+					Value:    "127.0.0.1",
+					Duration: "59m",
+					Type:     "ban",
+				},
+				{
+					ID:       int64(4),
+					Origin:   "test",
+					Scenario: "crowdsecurity/test",
+					Value:    "127.0.0.2",
+					Duration: "2h59",
+					Type:     "ban",
+				},
+				{
+					ID:       int64(8),
+					Origin:   "test",
+					Scenario: "crowdsecurity/test",
+					Value:    "127.0.0.2",
+					Duration: "2h59",
+					Type:     "captcha",
+				},
+			},
+		},
+		{
+			TestName:      "test startup with scenarios containing and not containing 2",
+			Method:        "GET",
+			Route:         "/v1/decisions/stream?startup=true&scenarios_containing=longest&scenarios_not_containing=ssh_bf,test",
+			CheckCodeOnly: false,
+			Code:          200,
+			LenNew:        1,
+			LenDeleted:    0,
+			AuthType:      APIKEY,
+			DelChecks:     []DecisionCheck{},
+			NewChecks: []DecisionCheck{
+				{
+					ID:       int64(3),
+					Origin:   "test",
+					Scenario: "crowdsecurity/longest",
+					Value:    "127.0.0.1",
+					Duration: "4h59",
+					Type:     "ban",
+				},
+			},
+		},
+		{
+			TestName:      "test startup with scenarios not containing",
+			Method:        "GET",
+			Route:         "/v1/decisions/stream?startup=true&scenarios_not_containing=ssh_bf",
+			CheckCodeOnly: false,
+			Code:          200,
+			LenNew:        3,
+			LenDeleted:    0,
+			AuthType:      APIKEY,
+			DelChecks:     []DecisionCheck{},
+
+			NewChecks: []DecisionCheck{
+				{
+					ID:       int64(3),
+					Origin:   "test",
+					Scenario: "crowdsecurity/longest",
+					Value:    "127.0.0.1",
+					Duration: "4h59",
+					Type:     "ban",
+				},
+				{
+					ID:       int64(4),
+					Origin:   "test",
+					Scenario: "crowdsecurity/test",
+					Value:    "127.0.0.2",
+					Duration: "2h59",
+					Type:     "ban",
+				},
+				{
+					ID:       int64(8),
+					Origin:   "test",
+					Scenario: "crowdsecurity/test",
+					Value:    "127.0.0.2",
+					Duration: "2h59",
+					Type:     "captcha",
+				},
+			},
+		},
+		{
+			TestName:      "test startup with multiple scenarios not containing",
+			Method:        "GET",
+			Route:         "/v1/decisions/stream?startup=true&scenarios_not_containing=ssh_bf,test",
+			CheckCodeOnly: false,
+			Code:          200,
+			LenNew:        1,
+			LenDeleted:    0,
+			AuthType:      APIKEY,
+			DelChecks:     []DecisionCheck{},
+
+			NewChecks: []DecisionCheck{
+				{
+					ID:       int64(3),
+					Origin:   "test",
+					Scenario: "crowdsecurity/longest",
+					Value:    "127.0.0.1",
+					Duration: "4h59",
+					Type:     "ban",
+				},
+			},
+		},
+		{
+			TestName:      "test startup with origins parameter",
+			Method:        "GET",
+			Route:         "/v1/decisions/stream?startup=true&origins=another_origin",
+			CheckCodeOnly: false,
+			Code:          200,
+			LenNew:        2,
+			LenDeleted:    0,
+			AuthType:      APIKEY,
+			DelChecks:     []DecisionCheck{},
+
+			NewChecks: []DecisionCheck{
+				{
+					ID:       int64(2),
+					Origin:   "another_origin",
+					Scenario: "crowdsecurity/ssh_bf",
+					Value:    "127.0.0.1",
+					Duration: "2h59",
+					Type:     "ban",
+				},
+				{
+					ID:       int64(7),
+					Origin:   "another_origin",
+					Scenario: "crowdsecurity/test",
+					Value:    "127.0.0.2",
+					Duration: "1h59",
+					Type:     "ban",
+				},
+			},
+		},
+		{
+			TestName:      "test startup with multiple origins parameter",
+			Method:        "GET",
+			Route:         "/v1/decisions/stream?startup=true&origins=another_origin,test",
+			CheckCodeOnly: false,
+			Code:          200,
+			LenNew:        3,
+			LenDeleted:    0,
+			AuthType:      APIKEY,
+			DelChecks:     []DecisionCheck{},
+
+			NewChecks: []DecisionCheck{
+				{
+					ID:       int64(3),
+					Origin:   "test",
+					Scenario: "crowdsecurity/longest",
+					Value:    "127.0.0.1",
+					Duration: "4h59",
+					Type:     "ban",
+				},
+				{
+					ID:       int64(4),
+					Origin:   "test",
+					Scenario: "crowdsecurity/test",
+					Value:    "127.0.0.2",
+					Duration: "2h59",
+					Type:     "ban",
+				},
+				{
+					ID:       int64(8),
+					Origin:   "test",
+					Scenario: "crowdsecurity/test",
+					Value:    "127.0.0.2",
+					Duration: "2h59",
+					Type:     "captcha",
+				},
+			},
+		},
+		{
+			TestName:      "test startup with unknown origins",
+			Method:        "GET",
+			Route:         "/v1/decisions/stream?startup=true&origins=unknown",
+			CheckCodeOnly: false,
+			Code:          200,
+			LenNew:        0,
+			LenDeleted:    0,
+			AuthType:      APIKEY,
+			DelChecks:     []DecisionCheck{},
+			NewChecks:     []DecisionCheck{},
+		},
+		{
+			TestName:      "delete decisions 3 (127.0.0.1)",
+			Method:        "DELETE",
+			Route:         "/v1/decisions/3",
+			CheckCodeOnly: true,
+			Code:          200,
+			LenNew:        0,
+			LenDeleted:    0,
+			AuthType:      PASSWORD,
+			DelChecks:     []DecisionCheck{},
+			NewChecks:     []DecisionCheck{},
+		},
+		{
+			TestName:      "check that 127.0.0.1 is not in deleted IP",
+			Method:        "GET",
+			Route:         "/v1/decisions/stream?startup=true",
+			CheckCodeOnly: false,
+			Code:          200,
+			LenNew:        3,
+			LenDeleted:    0,
+			AuthType:      APIKEY,
+			DelChecks:     []DecisionCheck{},
+			NewChecks: []DecisionCheck{
+				{
+					ID:       int64(2),
+					Origin:   "another_origin",
+					Scenario: "crowdsecurity/ssh_bf",
+					Value:    "127.0.0.1",
+					Duration: "2h59",
+					Type:     "ban",
+				},
+				{
+					ID:       int64(4),
+					Origin:   "test",
+					Scenario: "crowdsecurity/test",
+					Value:    "127.0.0.2",
+					Duration: "2h59",
+					Type:     "ban",
+				},
+				{
+					ID:       int64(8),
+					Origin:   "test",
+					Scenario: "crowdsecurity/test",
+					Value:    "127.0.0.2",
+					Duration: "2h59",
+					Type:     "captcha",
+				},
+			},
+		},
+		{
+			TestName:      "delete decisions 2 (127.0.0.1)",
+			Method:        "DELETE",
+			Route:         "/v1/decisions/2",
+			CheckCodeOnly: true,
+			Code:          200,
+			LenNew:        0,
+			LenDeleted:    0,
+			AuthType:      PASSWORD,
+			DelChecks:     []DecisionCheck{},
+			NewChecks:     []DecisionCheck{},
+		},
+		{
+			TestName:      "check that 127.0.0.1 is not in deleted IP",
+			Method:        "GET",
+			Route:         "/v1/decisions/stream?startup=true",
+			CheckCodeOnly: false,
+			Code:          200,
+			LenNew:        3,
+			LenDeleted:    0,
+			AuthType:      APIKEY,
+			DelChecks:     []DecisionCheck{},
+			NewChecks: []DecisionCheck{
+				{
+					ID:       int64(1),
+					Origin:   "test",
+					Scenario: "crowdsecurity/test",
+					Value:    "127.0.0.1",
+					Duration: "59",
+					Type:     "ban",
+				},
+				{
+					ID:       int64(4),
+					Origin:   "test",
+					Scenario: "crowdsecurity/test",
+					Value:    "127.0.0.2",
+					Duration: "2h59",
+					Type:     "ban",
+				},
+				{
+					ID:       int64(8),
+					Origin:   "test",
+					Scenario: "crowdsecurity/test",
+					Value:    "127.0.0.2",
+					Duration: "2h59",
+					Type:     "captcha",
+				},
+			},
+		},
+		{
+			TestName:      "delete decisions 1 (127.0.0.1)",
+			Method:        "DELETE",
+			Route:         "/v1/decisions/1",
+			CheckCodeOnly: true,
+			Code:          200,
+			LenNew:        0,
+			LenDeleted:    0,
+			AuthType:      PASSWORD,
+			DelChecks:     []DecisionCheck{},
+			NewChecks:     []DecisionCheck{},
+		},
+		{
+			TestName:      "127.0.0.1 should be in deleted now",
+			Method:        "GET",
+			Route:         "/v1/decisions/stream?startup=true",
+			CheckCodeOnly: false,
+			Code:          200,
+			LenNew:        2,
+			LenDeleted:    1,
+			AuthType:      APIKEY,
+			DelChecks: []DecisionCheck{
+				{
+					ID:       int64(1),
+					Origin:   "test",
+					Scenario: "crowdsecurity/test",
+					Value:    "127.0.0.1",
+					Duration: "-", // we check that the time is negative
+					Type:     "ban",
+				},
+			},
+			NewChecks: []DecisionCheck{
+				{
+					ID:       int64(4),
+					Origin:   "test",
+					Scenario: "crowdsecurity/test",
+					Value:    "127.0.0.2",
+					Duration: "2h59",
+					Type:     "ban",
+				},
+				{
+					ID:       int64(8),
+					Origin:   "test",
+					Scenario: "crowdsecurity/test",
+					Value:    "127.0.0.2",
+					Duration: "2h59",
+					Type:     "captcha",
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		runTest(lapi, test, t)
+	}
+}
 
-	// test filters both by scenarios_not_containing and scenarios_containing
-	w = lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true&scenarios_not_containing=ssh&scenarios_containing=ddos", emptyBody, "apikey")
-	decisions, code, err = readDecisionsStreamResp(w)
-	assert.Equal(t, err, nil)
-	assert.Equal(t, 200, code)
-	assert.Equal(t, len(decisions["deleted"]), 0)
-	assert.Equal(t, len(decisions["new"]), 1)
-	assert.Equal(t, decisions["new"][0].ID, int64(3))
+func TestStreamDecision(t *testing.T) {
+
+	/*
+		Create multiple alerts:
+		  - 3 alerts for 127.0.0.1 with ID 1/2/3   : Different duration / scenario / origin
+		  - 3 alerts for 127.0.0.2 with ID 4/5/6/7 : Different duration / scenario / origin
+	*/
+
+	// this test just init the stream with startup=true
+	preTests := []DecisionTest{
+		{
+			TestName:      "test startup",
+			Method:        "GET",
+			Route:         "/v1/decisions/stream?startup=true",
+			CheckCodeOnly: false,
+			Code:          200,
+			AuthType:      APIKEY,
+			LenNew:        0,
+			LenDeleted:    0,
+			DelChecks:     []DecisionCheck{},
+			NewChecks:     []DecisionCheck{},
+		},
+	}
+
+	tests := map[string][]DecisionTest{
+		"Test without parameter": {
+			{
+				TestName:      "get stream",
+				Method:        "GET",
+				Route:         "/v1/decisions/stream",
+				CheckCodeOnly: false,
+				Code:          200,
+				LenNew:        3,
+				LenDeleted:    0,
+				AuthType:      APIKEY,
+				DelChecks:     []DecisionCheck{},
+				NewChecks: []DecisionCheck{
+					{
+						ID:       int64(3),
+						Origin:   "test",
+						Scenario: "crowdsecurity/longest",
+						Value:    "127.0.0.1",
+						Duration: "4h59",
+						Type:     "ban",
+					},
+					{
+						ID:       int64(4),
+						Origin:   "test",
+						Scenario: "crowdsecurity/test",
+						Value:    "127.0.0.2",
+						Duration: "2h59",
+						Type:     "ban",
+					},
+					{
+						ID:       int64(8),
+						Origin:   "test",
+						Scenario: "crowdsecurity/test",
+						Value:    "127.0.0.2",
+						Duration: "2h59",
+						Type:     "captcha",
+					},
+				},
+			},
+			{
+				TestName:      "delete decisions 3 (127.0.0.1)",
+				Method:        "DELETE",
+				Route:         "/v1/decisions/3",
+				CheckCodeOnly: true,
+				Code:          200,
+				LenNew:        0,
+				LenDeleted:    0,
+				AuthType:      PASSWORD,
+				DelChecks:     []DecisionCheck{},
+				NewChecks:     []DecisionCheck{},
+			},
+			{
+				TestName:      "check that 127.0.0.1 is not in deleted IP",
+				Method:        "GET",
+				Route:         "/v1/decisions/stream",
+				CheckCodeOnly: false,
+				Code:          200,
+				LenNew:        0,
+				LenDeleted:    0,
+				AuthType:      APIKEY,
+				DelChecks:     []DecisionCheck{},
+				NewChecks:     []DecisionCheck{},
+			},
+			{
+				TestName:      "delete decisions 2 (127.0.0.1)",
+				Method:        "DELETE",
+				Route:         "/v1/decisions/2",
+				CheckCodeOnly: true,
+				Code:          200,
+				LenNew:        0,
+				LenDeleted:    0,
+				AuthType:      PASSWORD,
+				DelChecks:     []DecisionCheck{},
+				NewChecks:     []DecisionCheck{},
+			},
+			{
+				TestName:      "check that 127.0.0.1 is not in deleted IP",
+				Method:        "GET",
+				Route:         "/v1/decisions/stream",
+				CheckCodeOnly: false,
+				Code:          200,
+				LenNew:        0,
+				LenDeleted:    0,
+				AuthType:      APIKEY,
+				DelChecks:     []DecisionCheck{},
+				NewChecks:     []DecisionCheck{},
+			},
+			{
+				TestName:      "delete decisions 1 (127.0.0.1)",
+				Method:        "DELETE",
+				Route:         "/v1/decisions/1",
+				CheckCodeOnly: true,
+				Code:          200,
+				LenNew:        0,
+				LenDeleted:    0,
+				AuthType:      PASSWORD,
+				DelChecks:     []DecisionCheck{},
+				NewChecks:     []DecisionCheck{},
+			},
+			{
+				TestName:      "127.0.0.1 should be in deleted now",
+				Method:        "GET",
+				Route:         "/v1/decisions/stream",
+				CheckCodeOnly: false,
+				Code:          200,
+				LenNew:        0,
+				LenDeleted:    1,
+				AuthType:      APIKEY,
+				DelChecks: []DecisionCheck{
+					{
+						ID:       int64(1),
+						Origin:   "test",
+						Scenario: "crowdsecurity/test",
+						Value:    "127.0.0.1",
+						Duration: "-",
+
+						Type: "ban",
+					},
+				},
+				NewChecks: []DecisionCheck{},
+			},
+		},
+		"test with scenarios containing": {
+			{
+				TestName:      "get stream",
+				Method:        "GET",
+				Route:         "/v1/decisions/stream?scenarios_containing=ssh_bf",
+				CheckCodeOnly: false,
+				Code:          200,
+				LenNew:        2,
+				LenDeleted:    0,
+				AuthType:      APIKEY,
+				DelChecks:     []DecisionCheck{},
+				NewChecks: []DecisionCheck{
+					{
+						ID:       int64(2),
+						Origin:   "another_origin",
+						Scenario: "crowdsecurity/ssh_bf",
+						Value:    "127.0.0.1",
+						Duration: "2h59",
+						Type:     "ban",
+					},
+					{
+						ID:       int64(5),
+						Origin:   "test",
+						Scenario: "crowdsecurity/ssh_bf",
+						Value:    "127.0.0.2",
+						Duration: "2h59",
+						Type:     "ban",
+					},
+				},
+			},
+			{
+				TestName:      "delete decisions 3 (127.0.0.1)",
+				Method:        "DELETE",
+				Route:         "/v1/decisions/3",
+				CheckCodeOnly: true,
+				Code:          200,
+				LenNew:        0,
+				LenDeleted:    0,
+				AuthType:      PASSWORD,
+				DelChecks:     []DecisionCheck{},
+				NewChecks:     []DecisionCheck{},
+			},
+			{
+				TestName:      "check that 127.0.0.1 is not in deleted IP",
+				Method:        "GET",
+				Route:         "/v1/decisions/stream?scenarios_containing=ssh_bf",
+				CheckCodeOnly: false,
+				Code:          200,
+				LenNew:        0,
+				LenDeleted:    0,
+				AuthType:      APIKEY,
+				DelChecks:     []DecisionCheck{},
+				NewChecks:     []DecisionCheck{},
+			},
+			{
+				TestName:      "delete decisions 2 (127.0.0.1)",
+				Method:        "DELETE",
+				Route:         "/v1/decisions/2",
+				CheckCodeOnly: true,
+				Code:          200,
+				LenNew:        0,
+				LenDeleted:    0,
+				AuthType:      PASSWORD,
+				DelChecks:     []DecisionCheck{},
+				NewChecks:     []DecisionCheck{},
+			},
+			{
+				TestName:      "check that 127.0.0.1 is deleted (decision for ssh_bf was with ID 2)",
+				Method:        "GET",
+				Route:         "/v1/decisions/stream?scenarios_containing=ssh_bf",
+				CheckCodeOnly: false,
+				Code:          200,
+				LenNew:        0,
+				LenDeleted:    1,
+				AuthType:      APIKEY,
+				DelChecks: []DecisionCheck{
+					{
+						ID:       int64(2),
+						Origin:   "another_origin",
+						Scenario: "crowdsecurity/ssh_bf",
+						Value:    "127.0.0.1",
+						Duration: "-",
+
+						Type: "ban",
+					},
+				},
+				NewChecks: []DecisionCheck{},
+			},
+		},
+		"test with scenarios not containing": {
+			{
+				TestName:      "get stream",
+				Method:        "GET",
+				Route:         "/v1/decisions/stream?scenarios_not_containing=ssh_bf",
+				CheckCodeOnly: false,
+				Code:          200,
+				LenNew:        3,
+				LenDeleted:    0,
+				AuthType:      APIKEY,
+				DelChecks:     []DecisionCheck{},
+				NewChecks: []DecisionCheck{
+					{
+						ID:       int64(3),
+						Origin:   "test",
+						Scenario: "crowdsecurity/longest",
+						Value:    "127.0.0.1",
+						Duration: "4h59",
+						Type:     "ban",
+					},
+					{
+						ID:       int64(4),
+						Origin:   "test",
+						Scenario: "crowdsecurity/test",
+						Value:    "127.0.0.2",
+						Duration: "2h59",
+						Type:     "ban",
+					},
+					{
+						ID:       int64(8),
+						Origin:   "test",
+						Scenario: "crowdsecurity/test",
+						Value:    "127.0.0.2",
+						Duration: "2h59",
+						Type:     "captcha",
+					},
+				},
+			},
+			{
+				TestName:      "delete decisions 3 (127.0.0.1)",
+				Method:        "DELETE",
+				Route:         "/v1/decisions/3",
+				CheckCodeOnly: true,
+				Code:          200,
+				LenNew:        0,
+				LenDeleted:    0,
+				AuthType:      PASSWORD,
+				DelChecks:     []DecisionCheck{},
+				NewChecks:     []DecisionCheck{},
+			},
+			{
+				TestName:      "check that 127.0.0.1 is not in deleted IP",
+				Method:        "GET",
+				Route:         "/v1/decisions/stream?scenarios_not_containing=ssh_bf",
+				CheckCodeOnly: false,
+				Code:          200,
+				LenNew:        0,
+				LenDeleted:    0,
+				AuthType:      APIKEY,
+				DelChecks:     []DecisionCheck{},
+				NewChecks:     []DecisionCheck{},
+			},
+			{
+				TestName:      "delete decisions 2 (127.0.0.1)",
+				Method:        "DELETE",
+				Route:         "/v1/decisions/2",
+				CheckCodeOnly: true,
+				Code:          200,
+				LenNew:        0,
+				LenDeleted:    0,
+				AuthType:      PASSWORD,
+				DelChecks:     []DecisionCheck{},
+				NewChecks:     []DecisionCheck{},
+			},
+			{
+				TestName:      "check that 127.0.0.1 is not deleted",
+				Method:        "GET",
+				Route:         "/v1/decisions/stream?scenarios_not_containing=ssh_bf",
+				CheckCodeOnly: false,
+				Code:          200,
+				LenNew:        0,
+				LenDeleted:    0,
+				AuthType:      APIKEY,
+				DelChecks:     []DecisionCheck{},
+				NewChecks:     []DecisionCheck{},
+			},
+			{
+				TestName:      "delete decisions 1 (127.0.0.1)",
+				Method:        "DELETE",
+				Route:         "/v1/decisions/1",
+				CheckCodeOnly: true,
+				Code:          200,
+				LenNew:        0,
+				LenDeleted:    0,
+				AuthType:      PASSWORD,
+				DelChecks:     []DecisionCheck{},
+				NewChecks:     []DecisionCheck{},
+			},
+			{
+				TestName:      "check that 127.0.0.1 is deleted",
+				Method:        "GET",
+				Route:         "/v1/decisions/stream?scenarios_not_containing=ssh_bf",
+				CheckCodeOnly: false,
+				Code:          200,
+				LenNew:        0,
+				LenDeleted:    1,
+				AuthType:      APIKEY,
+				DelChecks: []DecisionCheck{
+					{
+						ID:       int64(1),
+						Origin:   "test",
+						Scenario: "crowdsecurity/test",
+						Value:    "127.0.0.1",
+						Duration: "-",
+
+						Type: "ban",
+					},
+				},
+				NewChecks: []DecisionCheck{},
+			},
+		},
+		"test with origins": {
+			{
+				TestName:      "get stream",
+				Method:        "GET",
+				Route:         "/v1/decisions/stream?origins=another_origin",
+				CheckCodeOnly: false,
+				Code:          200,
+				LenNew:        2,
+				LenDeleted:    0,
+				AuthType:      APIKEY,
+				DelChecks:     []DecisionCheck{},
+				NewChecks: []DecisionCheck{
+					{
+						ID:       int64(2),
+						Origin:   "another_origin",
+						Scenario: "crowdsecurity/ssh_bf",
+						Value:    "127.0.0.1",
+						Duration: "2h59",
+						Type:     "ban",
+					},
+					{
+						ID:       int64(7),
+						Origin:   "another_origin",
+						Scenario: "crowdsecurity/test",
+						Value:    "127.0.0.2",
+						Duration: "1h59",
+						Type:     "ban",
+					},
+				},
+			},
+			{
+				TestName:      "delete decisions 3 (127.0.0.1)",
+				Method:        "DELETE",
+				Route:         "/v1/decisions/3",
+				CheckCodeOnly: true,
+				Code:          200,
+				LenNew:        0,
+				LenDeleted:    0,
+				AuthType:      PASSWORD,
+				DelChecks:     []DecisionCheck{},
+				NewChecks:     []DecisionCheck{},
+			},
+			{
+				TestName:      "check that 127.0.0.1 is not in deleted IP",
+				Method:        "GET",
+				Route:         "/v1/decisions/stream?origins=another_origin",
+				CheckCodeOnly: false,
+				Code:          200,
+				LenNew:        0,
+				LenDeleted:    0,
+				AuthType:      APIKEY,
+				DelChecks:     []DecisionCheck{},
+				NewChecks:     []DecisionCheck{},
+			},
+			{
+				TestName:      "delete decisions 2 (127.0.0.1)",
+				Method:        "DELETE",
+				Route:         "/v1/decisions/2",
+				CheckCodeOnly: true,
+				Code:          200,
+				LenNew:        0,
+				LenDeleted:    0,
+				AuthType:      PASSWORD,
+				DelChecks:     []DecisionCheck{},
+				NewChecks:     []DecisionCheck{},
+			},
+			{
+				TestName:      "check that 127.0.0.1 is deleted",
+				Method:        "GET",
+				Route:         "/v1/decisions/stream?origins=another_origin",
+				CheckCodeOnly: false,
+				Code:          200,
+				LenNew:        0,
+				LenDeleted:    1,
+				AuthType:      APIKEY,
+				DelChecks: []DecisionCheck{
+					{
+						ID:       int64(2),
+						Origin:   "another_origin",
+						Scenario: "crowdsecurity/ssh_bf",
+						Value:    "127.0.0.1",
+						Duration: "-",
+
+						Type: "ban",
+					},
+				},
+				NewChecks: []DecisionCheck{},
+			},
+		},
+	}
+
+	// run tests for the stream
+	for testName, test := range tests {
+
+		// init a new LAPI
+		lapi := SetupLAPITest(t)
+
+		// run pre-test, mostly to init the stream
+		for _, test := range preTests {
+			runTest(lapi, test, t)
+		}
+		// insert decisions now that the stream is initiated
+		lapi.InsertAlertFromFile("./tests/alert_duplicate.json")
+
+		for _, oneTest := range test {
+			oneTest.TestName = fmt.Sprintf("%s (%s)", oneTest.TestName, testName)
+			runTest(lapi, oneTest, t)
+		}
+
+		// clean the db after each test
+		os.Remove(lapi.DBConfig.DbPath)
+	}
+}
 
-	// test filter by origin
-	w = lapi.RecordResponse("GET", "/v1/decisions/stream?startup=true&origins=test1,test2", emptyBody, "apikey")
-	decisions, code, err = readDecisionsStreamResp(w)
-	assert.Equal(t, err, nil)
-	assert.Equal(t, 200, code)
-	assert.Equal(t, len(decisions["deleted"]), 0)
-	assert.Equal(t, len(decisions["new"]), 2)
-	assert.Equal(t, decisions["new"][0].ID, int64(1))
-	assert.Equal(t, decisions["new"][1].ID, int64(2))
+func runTest(lapi LAPI, test DecisionTest, t *testing.T) {
+	w := lapi.RecordResponse(test.Method, test.Route, emptyBody, test.AuthType)
+	assert.Equal(t, test.Code, w.Code)
+	if test.CheckCodeOnly {
+		return
+	}
+	decisions, _, err := readDecisionsStreamResp(w)
+	assert.Equal(t, nil, err)
+	assert.Equal(t, test.LenDeleted, len(decisions["deleted"]), fmt.Sprintf("'%s': len(deleted)", test.TestName))
+	assert.Equal(t, test.LenNew, len(decisions["new"]), fmt.Sprintf("'%s': len(new)", test.TestName))
+
+	for i, check := range test.NewChecks {
+		assert.Equal(t, check.ID, decisions["new"][i].ID, fmt.Sprintf("'%s' (idx: %d): field: ID", test.TestName, i))
+		assert.Equal(t, check.Origin, *decisions["new"][i].Origin, fmt.Sprintf("'%s' (idx: %d): field: Origin", test.TestName, i))
+		assert.Equal(t, check.Scenario, *decisions["new"][i].Scenario, fmt.Sprintf("'%s' (idx: %d): field: Scenario", test.TestName, i))
+		assert.Equal(t, check.Value, *decisions["new"][i].Value, fmt.Sprintf("'%s' (idx: %d): field: Value", test.TestName, i))
+		assert.Equal(t, check.Type, *decisions["new"][i].Type, fmt.Sprintf("'%s' (idx: %d): field: Type", test.TestName, i))
+		assert.Contains(t, *decisions["new"][i].Duration, check.Duration, fmt.Sprintf("'%s' (idx: %d): field: Duration", test.TestName, i))
+	}
+
+	for i, check := range test.DelChecks {
+		assert.Equal(t, check.ID, decisions["deleted"][i].ID, fmt.Sprintf("'%s' (idx: %d): field: ID", test.TestName, i))
+		assert.Equal(t, check.Origin, *decisions["deleted"][i].Origin, fmt.Sprintf("'%s' (idx: %d): field: Origin", test.TestName, i))
+		assert.Equal(t, check.Scenario, *decisions["deleted"][i].Scenario, fmt.Sprintf("'%s' (idx: %d): field: Scenario", test.TestName, i))
+		assert.Equal(t, check.Value, *decisions["deleted"][i].Value, fmt.Sprintf("'%s' (idx: %d): field: Value", test.TestName, i))
+		assert.Equal(t, check.Type, *decisions["deleted"][i].Type, fmt.Sprintf("'%s' (idx: %d): field: Type", test.TestName, i))
+		assert.Contains(t, *decisions["deleted"][i].Duration, check.Duration, fmt.Sprintf("'%s' (idx: %d): field: Duration", test.TestName, i))
+	}
 }

pkg/apiserver/tests/alert_duplicate.json

@@ -0,0 +1,266 @@
+[
+    {
+        "id": 42,
+        "machine_id": "test",
+        "capacity": 1,
+        "created_at": "2020-10-09T10:00:10Z",
+        "decisions": [
+            {
+                "duration": "1h",
+                "origin": "test",
+                "scenario": "crowdsecurity/test",
+                "scope": "Ip",
+                "value": "127.0.0.1",
+                "type": "ban"
+            }
+        ],
+        "source": {
+            "ip": "127.0.0.1",
+            "range": "127.0.0.1/32",
+            "scope": "ip",
+            "value": "127.0.0.1"
+        },
+        "Events": [
+        ],
+        "events_count": 1,
+        "leakspeed": "0.5s",
+        "message": "test",
+        "scenario_hash": "hashtest",
+        "scenario_version": "v1",
+        "simulated": false,
+        "scenario": "crowdsecurity/test",
+        "start_at": "2020-10-09T10:00:01Z",
+        "stop_at": "2020-10-09T10:00:05Z"
+    },
+    {
+        "id": 44,
+        "machine_id": "test",
+        "created_at": "2020-10-09T10:00:10Z",
+        "decisions": [
+            {
+                "duration": "3h",
+                "origin": "another_origin",
+                "scenario": "crowdsecurity/ssh_bf",
+                "scope": "Ip",
+                "value": "127.0.0.1",
+                "type": "ban"
+            }
+        ],
+        "source": {
+            "ip": "127.0.0.1",
+            "range": "127.0.0.1/32",
+            "scope": "ip",
+            "value": "127.0.0.1"
+        },
+        "Events": [
+        ],
+        "events_count": 1,
+        "leakspeed": "0.5s",
+        "message": "test",
+        "scenario_hash": "hashtest",
+        "scenario_version": "v1",
+        "simulated": false,
+        "capacity": 1,
+        "scenario": "crowdsecurity/ssh_bf",
+        "start_at": "2020-10-09T10:00:01Z",
+        "stop_at": "2020-10-09T10:00:05Z"
+    },
+    {
+        "id": 45,
+        "machine_id": "test",
+        "created_at": "2020-10-09T10:00:10Z",
+        "decisions": [
+            {
+                "duration": "5h",
+                "origin": "test",
+                "scenario": "crowdsecurity/longest",
+                "scope": "Ip",
+                "value": "127.0.0.1",
+                "type": "ban"
+            }
+        ],
+        "source": {
+            "ip": "127.0.0.1",
+            "range": "127.0.0.1/32",
+            "scope": "ip",
+            "value": "127.0.0.1"
+        },
+        "Events": [
+        ],
+        "events_count": 1,
+        "leakspeed": "0.5s",
+        "message": "test",
+        "scenario_hash": "hashtest",
+        "scenario_version": "v1",
+        "simulated": false,
+        "capacity": 1,
+        "scenario": "crowdsecurity/longest",
+        "start_at": "2020-10-09T10:00:01Z",
+        "stop_at": "2020-10-09T10:00:05Z"
+    },
+    {
+        "id": 46,
+        "machine_id": "test",
+        "created_at": "2020-10-09T10:00:10Z",
+        "decisions": [
+            {
+                "duration": "3h",
+                "origin": "test",
+                "scenario": "crowdsecurity/test",
+                "scope": "Ip",
+                "value": "127.0.0.2",
+                "type": "ban"
+            }
+        ],
+        "source": {
+            "ip": "127.0.0.2",
+            "range": "127.0.0.2/32",
+            "scope": "ip",
+            "value": "127.0.0.2"
+        },
+        "Events": [
+        ],
+        "events_count": 1,
+        "leakspeed": "0.5s",
+        "message": "test",
+        "scenario_hash": "hashtest",
+        "scenario_version": "v1",
+        "simulated": false,
+        "capacity": 1,
+        "scenario": "crowdsecurity/test",
+        "start_at": "2020-10-09T10:00:01Z",
+        "stop_at": "2020-10-09T10:00:05Z"
+    },
+    {
+        "id": 47,
+        "machine_id": "test",
+        "created_at": "2020-10-09T10:00:10Z",
+        "decisions": [
+            {
+                "duration": "3h",
+                "origin": "test",
+                "scenario": "crowdsecurity/ssh_bf",
+                "scope": "Ip",
+                "value": "127.0.0.2",
+                "type": "ban"
+            }
+        ],
+        "source": {
+            "ip": "127.0.0.2",
+            "range": "127.0.0.2/32",
+            "scope": "ip",
+            "value": "127.0.0.2"
+        },
+        "Events": [
+        ],
+        "events_count": 1,
+        "leakspeed": "0.5s",
+        "message": "test",
+        "scenario_hash": "hashtest",
+        "scenario_version": "v1",
+        "simulated": false,
+        "capacity": 1,
+        "scenario": "crowdsecurity/ssh_bf",
+        "start_at": "2020-10-09T10:00:01Z",
+        "stop_at": "2020-10-09T10:00:05Z"
+    },
+    {
+        "id": 48,
+        "machine_id": "test",
+        "created_at": "2020-10-09T10:00:10Z",
+        "decisions": [
+            {
+                "duration": "1h",
+                "origin": "test",
+                "scenario": "crowdsecurity/ssh_bf",
+                "scope": "Ip",
+                "value": "127.0.0.2",
+                "type": "ban"
+            }
+        ],
+        "source": {
+            "ip": "127.0.0.2",
+            "range": "127.0.0.2/32",
+            "scope": "ip",
+            "value": "127.0.0.2"
+        },
+        "Events": [
+        ],
+        "events_count": 1,
+        "leakspeed": "0.5s",
+        "message": "test",
+        "scenario_hash": "hashtest",
+        "scenario_version": "v1",
+        "simulated": false,
+        "capacity": 1,
+        "scenario": "crowdsecurity/ssh_bf",
+        "start_at": "2020-10-09T10:00:01Z",
+        "stop_at": "2020-10-09T10:00:05Z"
+    },
+    {
+        "id": 49,
+        "machine_id": "test",
+        "created_at": "2020-10-09T10:00:10Z",
+        "decisions": [
+            {
+                "duration": "2h",
+                "origin": "another_origin",
+                "scenario": "crowdsecurity/test",
+                "scope": "Ip",
+                "value": "127.0.0.2",
+                "type": "ban"
+            }
+        ],
+        "source": {
+            "ip": "127.0.0.2",
+            "range": "127.0.0.2/32",
+            "scope": "ip",
+            "value": "127.0.0.2"
+        },
+        "Events": [
+        ],
+        "events_count": 1,
+        "leakspeed": "0.5s",
+        "message": "test",
+        "scenario_hash": "hashtest",
+        "scenario_version": "v1",
+        "simulated": false,
+        "capacity": 1,
+        "scenario": "crowdsecurity/test",
+        "start_at": "2020-10-09T10:00:01Z",
+        "stop_at": "2020-10-09T10:00:05Z"
+    },
+    {
+        "id": 50,
+        "machine_id": "test",
+        "created_at": "2020-10-09T10:00:10Z",
+        "decisions": [
+            {
+                "duration": "3h",
+                "origin": "test",
+                "scenario": "crowdsecurity/test",
+                "scope": "Ip",
+                "value": "127.0.0.2",
+                "type": "captcha"
+            }
+        ],
+        "source": {
+            "ip": "127.0.0.2",
+            "range": "127.0.0.2/32",
+            "scope": "ip",
+            "value": "127.0.0.2"
+        },
+        "Events": [
+        ],
+        "events_count": 1,
+        "leakspeed": "0.5s",
+        "message": "test",
+        "scenario_hash": "hashtest",
+        "scenario_version": "v1",
+        "simulated": false,
+        "capacity": 1,
+        "scenario": "crowdsecurity/test",
+        "start_at": "2020-10-09T10:00:01Z",
+        "stop_at": "2020-10-09T10:00:05Z"
+    }
+]

pkg/database/decisions.go

@@ -15,16 +15,19 @@ import (
 	"github.com/pkg/errors"
 )
 
-func BuildDecisionRequestWithFilter(query *ent.DecisionQuery, filter map[string][]string) (*ent.DecisionQuery, error) {
+func BuildDecisionRequestWithFilter(query *ent.DecisionQuery, filter map[string][]string) (*ent.DecisionQuery, []*sql.Predicate, error) {
 
 	var err error
 	var start_ip, start_sfx, end_ip, end_sfx int64
 	var ip_sz int
 	var contains bool = true
-	/*if contains is true, return bans that *contains* the given value (value is the inner)
-	  else, return bans that are *contained* by the given value (value is the outer)*/
 
-	/*the simulated filter is a bit different : if it's not present *or* set to false, specifically exclude records with simulated to true */
+	// contains == true -> return bans that *contain* the given value (value is the inner)
+	// contains == false or missing -> return bans *contained* in the given value (value is the outer)
+
+	// simulated == true -> include simulated rows
+	// simulated == false or missing -> exclude simulated rows
+
 	if v, ok := filter["simulated"]; ok {
 		if v[0] == "false" {
 			query = query.Where(decision.SimulatedEQ(false))
@@ -33,13 +36,14 @@ func BuildDecisionRequestWithFilter(query *ent.DecisionQuery, filter map[string]
 	} else {
 		query = query.Where(decision.SimulatedEQ(false))
 	}
-
+	t := sql.Table(decision.Table)
+	joinPredicate := make([]*sql.Predicate, 0)
 	for param, value := range filter {
 		switch param {
 		case "contains":
 			contains, err = strconv.ParseBool(value[0])
 			if err != nil {
-				return nil, errors.Wrapf(InvalidFilter, "invalid contains value : %s", err)
+				return nil, nil, errors.Wrapf(InvalidFilter, "invalid contains value : %s", err)
 			}
 		case "scopes":
 			scopes := strings.Split(value[0], ",")
@@ -64,9 +68,24 @@ func BuildDecisionRequestWithFilter(query *ent.DecisionQuery, filter map[string]
 			query = query.Where(
 				decision.OriginIn(strings.Split(value[0], ",")...),
 			)
+			origins := strings.Split(value[0], ",")
+			originsContainsPredicate := make([]*sql.Predicate, 0)
+			for _, origin := range origins {
+				pred := sql.EqualFold(t.C(decision.FieldOrigin), origin)
+				originsContainsPredicate = append(originsContainsPredicate, pred)
+			}
+			joinPredicate = append(joinPredicate, sql.Or(originsContainsPredicate...))
 		case "scenarios_containing":
 			predicates := decisionPredicatesFromStr(value[0], decision.ScenarioContainsFold)
 			query = query.Where(decision.Or(predicates...))
+
+			scenarios := strings.Split(value[0], ",")
+			scenariosContainsPredicate := make([]*sql.Predicate, 0)
+			for _, scenario := range scenarios {
+				pred := sql.ContainsFold(t.C(decision.FieldScenario), scenario)
+				scenariosContainsPredicate = append(scenariosContainsPredicate, pred)
+			}
+			joinPredicate = append(joinPredicate, sql.Or(scenariosContainsPredicate...))
 		case "scenarios_not_containing":
 			predicates := decisionPredicatesFromStr(value[0], decision.ScenarioContainsFold)
 			query = query.Where(decision.Not(
@@ -74,10 +93,17 @@ func BuildDecisionRequestWithFilter(query *ent.DecisionQuery, filter map[string]
 					predicates...,
 				),
 			))
+			scenarios := strings.Split(value[0], ",")
+			scenariosContainsPredicate := make([]*sql.Predicate, 0)
+			for _, scenario := range scenarios {
+				pred := sql.ContainsFold(t.C(decision.FieldScenario), scenario)
+				scenariosContainsPredicate = append(scenariosContainsPredicate, sql.Not(pred))
+			}
+			joinPredicate = append(joinPredicate, sql.Or(scenariosContainsPredicate...))
 		case "ip", "range":
 			ip_sz, start_ip, start_sfx, end_ip, end_sfx, err = types.Addr2Ints(value[0])
 			if err != nil {
-				return nil, errors.Wrapf(InvalidIPOrRange, "unable to convert '%s' to int: %s", value[0], err)
+				return nil, nil, errors.Wrapf(InvalidIPOrRange, "unable to convert '%s' to int: %s", value[0], err)
 			}
 		}
 	}
@@ -149,9 +175,9 @@ func BuildDecisionRequestWithFilter(query *ent.DecisionQuery, filter map[string]
 			))
 		}
 	} else if ip_sz != 0 {
-		return nil, errors.Wrapf(InvalidFilter, "Unknown ip size %d", ip_sz)
+		return nil, nil, errors.Wrapf(InvalidFilter, "Unknown ip size %d", ip_sz)
 	}
-	return query, nil
+	return query, joinPredicate, nil
 }
 
 func (c *Client) QueryDecisionWithFilter(filter map[string][]string) ([]*ent.Decision, error) {
@@ -161,7 +187,7 @@ func (c *Client) QueryDecisionWithFilter(filter map[string][]string) ([]*ent.Dec
 	decisions := c.Ent.Decision.Query().
 		Where(decision.UntilGTE(time.Now().UTC()))
 
-	decisions, err = BuildDecisionRequestWithFilter(decisions, filter)
+	decisions, _, err = BuildDecisionRequestWithFilter(decisions, filter)
 	if err != nil {
 		return []*ent.Decision{}, err
 	}
@@ -185,86 +211,89 @@ func (c *Client) QueryDecisionWithFilter(filter map[string][]string) ([]*ent.Dec
 	return data, nil
 }
 
-// ent translation of https://stackoverflow.com/a/28090544
-func longestDecisionForScopeTypeValue(s *sql.Selector) {
-	t := sql.Table(decision.Table)
-	s.LeftJoin(t).OnP(sql.And(
-		sql.ColumnsEQ(
-			t.C(decision.FieldValue),
-			s.C(decision.FieldValue),
-		),
-		sql.ColumnsEQ(
-			t.C(decision.FieldType),
-			s.C(decision.FieldType),
-		),
-		sql.ColumnsEQ(
-			t.C(decision.FieldScope),
-			s.C(decision.FieldScope),
-		),
-		sql.ColumnsGT(
-			t.C(decision.FieldUntil),
-			s.C(decision.FieldUntil),
-		),
-	))
-	s.Where(
-		sql.IsNull(
-			t.C(decision.FieldUntil),
-		),
-	)
-}
-
 func (c *Client) QueryAllDecisionsWithFilters(filters map[string][]string) ([]*ent.Decision, error) {
 	query := c.Ent.Decision.Query().Where(
 		decision.UntilGT(time.Now().UTC()),
-		longestDecisionForScopeTypeValue,
 	)
-	query, err := BuildDecisionRequestWithFilter(query, filters)
-
+	query, _, err := BuildDecisionRequestWithFilter(query, filters)
 	if err != nil {
 		c.Log.Warningf("QueryAllDecisionsWithFilters : %s", err)
 		return []*ent.Decision{}, errors.Wrap(QueryFail, "get all decisions with filters")
 	}
 
-	data, err := query.All(c.CTX)
+	//Order is *very* important, the dedup assumes that decisions are sorted per IP and per time left
+	data, err := query.Order(ent.Asc(decision.FieldValue), ent.Desc(decision.FieldUntil)).All(c.CTX)
 	if err != nil {
 		c.Log.Warningf("QueryAllDecisionsWithFilters : %s", err)
 		return []*ent.Decision{}, errors.Wrap(QueryFail, "get all decisions with filters")
 	}
+
 	return data, nil
 }
 
 func (c *Client) QueryExpiredDecisionsWithFilters(filters map[string][]string) ([]*ent.Decision, error) {
+	now := time.Now().UTC()
 	query := c.Ent.Decision.Query().Where(
 		decision.UntilLT(time.Now().UTC()),
-		longestDecisionForScopeTypeValue,
 	)
-	query, err := BuildDecisionRequestWithFilter(query, filters)
-
+	query, predicates, err := BuildDecisionRequestWithFilter(query, filters)
 	if err != nil {
 		c.Log.Warningf("QueryExpiredDecisionsWithFilters : %s", err)
 		return []*ent.Decision{}, errors.Wrap(QueryFail, "get expired decisions with filters")
 	}
-	data, err := query.All(c.CTX)
+	query = query.Where(func(s *sql.Selector) {
+		t := sql.Table(decision.Table)
+
+		subQuery := sql.Select(t.C(decision.FieldValue)).From(t).Where(sql.GT(t.C(decision.FieldUntil), now))
+		for _, predicate := range predicates {
+			subQuery.Where(predicate)
+		}
+		s.Where(
+			sql.NotIn(
+				s.C(decision.FieldValue),
+				subQuery,
+			),
+		)
+	})
+
+	data, err := query.Order(ent.Asc(decision.FieldValue), ent.Desc(decision.FieldUntil)).All(c.CTX)
 	if err != nil {
 		c.Log.Warningf("QueryExpiredDecisionsWithFilters : %s", err)
 		return []*ent.Decision{}, errors.Wrap(QueryFail, "expired decisions")
 	}
+
 	return data, nil
 }
 
 func (c *Client) QueryExpiredDecisionsSinceWithFilters(since time.Time, filters map[string][]string) ([]*ent.Decision, error) {
+	now := time.Now().UTC()
+
 	query := c.Ent.Decision.Query().Where(
-		decision.UntilLT(time.Now().UTC()),
+		decision.UntilLT(now),
 		decision.UntilGT(since),
-		longestDecisionForScopeTypeValue,
 	)
-	query, err := BuildDecisionRequestWithFilter(query, filters)
+	query, predicates, err := BuildDecisionRequestWithFilter(query, filters)
 	if err != nil {
 		c.Log.Warningf("QueryExpiredDecisionsSinceWithFilters : %s", err)
 		return []*ent.Decision{}, errors.Wrap(QueryFail, "expired decisions with filters")
 	}
 
-	data, err := query.All(c.CTX)
+	query = query.Where(func(s *sql.Selector) {
+		t := sql.Table(decision.Table)
+
+		subQuery := sql.Select(t.C(decision.FieldValue)).From(t).Where(sql.GT(t.C(decision.FieldUntil), now))
+		for _, predicate := range predicates {
+			subQuery.Where(predicate)
+		}
+		s.Where(
+			sql.NotIn(
+				s.C(decision.FieldValue),
+				subQuery,
+			),
+		)
+	})
+
+	data, err := query.Order(ent.Asc(decision.FieldValue), ent.Desc(decision.FieldUntil)).All(c.CTX)
 	if err != nil {
 		c.Log.Warningf("QueryExpiredDecisionsSinceWithFilters : %s", err)
 		return []*ent.Decision{}, errors.Wrap(QueryFail, "expired decisions with filters")
@@ -277,14 +306,15 @@ func (c *Client) QueryNewDecisionsSinceWithFilters(since time.Time, filters map[
 	query := c.Ent.Decision.Query().Where(
 		decision.CreatedAtGT(since),
 		decision.UntilGT(time.Now().UTC()),
-		longestDecisionForScopeTypeValue,
 	)
-	query, err := BuildDecisionRequestWithFilter(query, filters)
+	query, _, err := BuildDecisionRequestWithFilter(query, filters)
 	if err != nil {
-		c.Log.Warningf("QueryNewDecisionsSinceWithFilters : %s", err)
-		return []*ent.Decision{}, errors.Wrapf(QueryFail, "new decisions since '%s'", since.String())
+		c.Log.Warningf("BuildDecisionRequestWithFilter : %s", err)
+		return []*ent.Decision{}, errors.Wrap(QueryFail, "expired decisions with filters")
 	}
-	data, err := query.All(c.CTX)
+
+	//Order is *very* important, the dedup assumes that decisions are sorted per IP and per time left
+	data, err := query.Order(ent.Asc(decision.FieldValue), ent.Desc(decision.FieldUntil)).All(c.CTX)
 	if err != nil {
 		c.Log.Warningf("QueryNewDecisionsSinceWithFilters : %s", err)
 		return []*ent.Decision{}, errors.Wrapf(QueryFail, "new decisions since '%s'", since.String())
@@ -521,17 +551,17 @@ func (c *Client) SoftDeleteDecisionsWithFilter(filter map[string][]string) (stri
 }
 
 //SoftDeleteDecisionByID set the expiration of a decision to now()
-func (c *Client) SoftDeleteDecisionByID(decisionID int) error {
+func (c *Client) SoftDeleteDecisionByID(decisionID int) (int, error) {
 	nbUpdated, err := c.Ent.Decision.Update().Where(decision.IDEQ(decisionID)).SetUntil(time.Now().UTC()).Save(c.CTX)
 	if err != nil || nbUpdated == 0 {
 		c.Log.Warningf("SoftDeleteDecisionByID : %v (nb soft deleted: %d)", err, nbUpdated)
-		return errors.Wrapf(DeleteFail, "decision with id '%d' doesn't exist", decisionID)
+		return 0, errors.Wrapf(DeleteFail, "decision with id '%d' doesn't exist", decisionID)
 	}
 
 	if nbUpdated == 0 {
-		return ItemNotFound
+		return 0, ItemNotFound
 	}
-	return nil
+	return nbUpdated, nil
 }
 
 func decisionPredicatesFromStr(s string, predicateFunc func(string) predicate.Decision) []predicate.Decision {

tests/bats/99_lapi-stream-mode-scenario.bats

@@ -0,0 +1,233 @@
+#!/usr/bin/env bats
+# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
+
+set -u
+
+setup_file() {
+    load "../lib/setup_file.sh"
+    ./instance-data load
+    ./instance-crowdsec start
+    API_KEY=$(cscli bouncers add testbouncer -o raw)
+    export API_KEY
+    CROWDSEC_API_URL="http://localhost:8080"
+    export CROWDSEC_API_URL
+}
+
+teardown_file() {
+    load "../lib/teardown_file.sh"
+}
+
+setup() {
+    load "../lib/setup.sh"
+    skip
+}
+
+#----------
+
+api() {
+    URI="$1"
+    curl -s -H "X-Api-Key:${API_KEY}" "${CROWDSEC_API_URL}${URI}"
+}
+
+output_new_decisions() {
+    jq -c '.new | map(select(.origin!="CAPI")) | .[] | del(.id) | (.. | .duration?) |= capture("(?<d>[[:digit:]]+h[[:digit:]]+m)").d' <(output) | sort
+}
+
+
+@test "${FILE} adding decisions with different duration, scenario, origin" {
+    # origin: test
+    run -0 cscli decisions add -i 127.0.0.1 -d 1h -R crowdsecurity/test
+    ./instance-crowdsec stop
+    run -0 ./instance-db exec_sql "update decisions set origin='test' where origin='cscli'"
+    ./instance-crowdsec start
+
+    run -0 cscli decisions add -i 127.0.0.1 -d 3h -R crowdsecurity/ssh_bf
+    ./instance-crowdsec stop
+    run -0 ./instance-db exec_sql "update decisions set origin='another_origin' where origin='cscli'"
+    ./instance-crowdsec start
+
+    run -0 cscli decisions add -i 127.0.0.1 -d 5h -R crowdsecurity/longest
+    run -0 cscli decisions add -i 127.0.0.2 -d 3h -R crowdsecurity/test
+    run -0 cscli decisions add -i 127.0.0.2 -d 3h -R crowdsecurity/ssh_bf
+    run -0 cscli decisions add -i 127.0.0.2 -d 1h -R crowdsecurity/ssh_bf
+    ./instance-crowdsec stop
+    run -0 ./instance-db exec_sql "update decisions set origin='test' where origin='cscli'"
+    ./instance-crowdsec start
+
+    # origin: another_origin
+    run -0 cscli decisions add -i 127.0.0.2 -d 2h -R crowdsecurity/test
+    ./instance-crowdsec stop
+    run -0 ./instance-db exec_sql "update decisions set origin='another_origin' where origin='cscli'"
+    ./instance-crowdsec start
+}
+
+@test "${FILE} test startup" {
+    run -0 api "/v1/decisions/stream?startup=true"
+    run -0 output_new_decisions
+    assert_output - <<-EOT
+	{"duration":"2h59m","origin":"test","scenario":"crowdsecurity/test","scope":"Ip","type":"ban","value":"127.0.0.2"}
+	{"duration":"4h59m","origin":"test","scenario":"crowdsecurity/longest","scope":"Ip","type":"ban","value":"127.0.0.1"}
+	EOT
+}
+
+@test "${FILE} test startup with scenarios containing" {
+    run -0 api "/v1/decisions/stream?startup=true&scenarios_containing=ssh_bf"
+    run -0 output_new_decisions
+    assert_output - <<-EOT
+	{"duration":"2h59m","origin":"another_origin","scenario":"crowdsecurity/ssh_bf","scope":"Ip","type":"ban","value":"127.0.0.1"}
+	{"duration":"2h59m","origin":"test","scenario":"crowdsecurity/ssh_bf","scope":"Ip","type":"ban","value":"127.0.0.2"}
+	EOT
+}
+
+@test "${FILE} test startup with multiple scenarios containing" {
+    run -0 api "/v1/decisions/stream?startup=true&scenarios_containing=ssh_bf,test"
+    run -0 output_new_decisions
+    assert_output - <<-EOT
+	{"duration":"2h59m","origin":"another_origin","scenario":"crowdsecurity/ssh_bf","scope":"Ip","type":"ban","value":"127.0.0.1"}
+	{"duration":"2h59m","origin":"test","scenario":"crowdsecurity/test","scope":"Ip","type":"ban","value":"127.0.0.2"}
+	EOT
+}
+
+@test "${FILE} test startup with unknown scenarios containing" {
+    run -0 api "/v1/decisions/stream?startup=true&scenarios_containing=unknown"
+    assert_output '{"deleted":null,"new":null}'
+}
+
+@test "${FILE} test startup with scenarios containing and not containing" {
+    run -0 api "/v1/decisions/stream?startup=true&scenarios_containing=test&scenarios_not_containing=ssh_bf"
+    run -0 output_new_decisions
+    assert_output - <<-EOT
+	{"duration":"2h59m","origin":"test","scenario":"crowdsecurity/test","scope":"Ip","type":"ban","value":"127.0.0.2"}
+	{"origin":"test","scenario":"crowdsecurity/test","scope":"Ip","type":"ban","value":"127.0.0.1"}
+	EOT
+}
+
+@test "${FILE} test startup with scenarios containing and not containing 2" {
+    run -0 api "/v1/decisions/stream?startup=true&scenarios_containing=longest&scenarios_not_containing=ssh_bf,test"
+    run -0 output_new_decisions
+    assert_output - <<-EOT
+	{"duration":"4h59m","origin":"test","scenario":"crowdsecurity/longest","scope":"Ip","type":"ban","value":"127.0.0.1"}
+	EOT
+}
+
+@test "${FILE} test startup with scenarios not containing" {
+    run -0 api "/v1/decisions/stream?startup=true&scenarios_not_containing=ssh_bf"
+    run -0 output_new_decisions
+    assert_output - <<-EOT
+	{"duration":"2h59m","origin":"test","scenario":"crowdsecurity/test","scope":"Ip","type":"ban","value":"127.0.0.2"}
+	{"duration":"4h59m","origin":"test","scenario":"crowdsecurity/longest","scope":"Ip","type":"ban","value":"127.0.0.1"}
+	EOT
+}
+
+@test "${FILE} test startup with multiple scenarios not containing" {
+    run -0 api "/v1/decisions/stream?startup=true&scenarios_not_containing=ssh_bf,test"
+    run -0 output_new_decisions
+    assert_output - <<-EOT
+	{"duration":"4h59m","origin":"test","scenario":"crowdsecurity/longest","scope":"Ip","type":"ban","value":"127.0.0.1"}
+	EOT
+}
+
+@test "${FILE} test startup with origins parameter" {
+    run -0 api "/v1/decisions/stream?startup=true&origins=another_origin"
+    run -0 output_new_decisions
+    assert_output - <<-EOT
+	{"duration":"1h59m","origin":"another_origin","scenario":"crowdsecurity/test","scope":"Ip","type":"ban","value":"127.0.0.2"}
+	{"duration":"2h59m","origin":"another_origin","scenario":"crowdsecurity/ssh_bf","scope":"Ip","type":"ban","value":"127.0.0.1"}
+	EOT
+}
+
+@test "${FILE} test startup with multiple origins parameter" {
+    run -0 api "/v1/decisions/stream?startup=true&origins=another_origin,test"
+    run -0 output_new_decisions
+    assert_output - <<-EOT
+	{"duration":"2h59m","origin":"test","scenario":"crowdsecurity/test","scope":"Ip","type":"ban","value":"127.0.0.2"}
+	{"duration":"4h59m","origin":"test","scenario":"crowdsecurity/longest","scope":"Ip","type":"ban","value":"127.0.0.1"}
+	EOT
+}
+
+@test "${FILE} test startup with unknown origins" {
+    run -0 api "/v1/decisions/stream?startup=true&origins=unknown"
+    assert_output '{"deleted":null,"new":null}'
+}
+
+#@test "${FILE} delete decision 3 (127.0.0.1)" {
+#
+#        {
+#            TestName:      "delete decisions 3 (127.0.0.1)",
+#            Method:        "DELETE",
+#            Route:         "/v1/decisions/3",
+#            CheckCodeOnly: true,
+#            Code:          200,
+#            LenNew:        0,
+#            LenDeleted:    0,
+#            AuthType:      PASSWORD,
+#            DelChecks:     []DecisionCheck{},
+#            NewChecks:     []DecisionCheck{},
+#       TestName:      "check that 127.0.0.1 is not in deleted IP",
+#            Method:        "GET",
+#            Route:         "/v1/decisions/stream?startup=true",
+#            CheckCodeOnly: false,
+#            Code:          200,
+#            LenNew:        2,
+#            LenDeleted:    0,
+#            AuthType:      APIKEY,
+#            DelChecks:     []DecisionCheck{},
+#            NewChecks:     []DecisionCheck{},
+#        },
+#        {
+#            TestName:      "delete decisions 2 (127.0.0.1)",
+#            Method:        "DELETE",
+#            Route:         "/v1/decisions/2",
+#            CheckCodeOnly: true,
+#            Code:          200,
+#            LenNew:        0,
+#            LenDeleted:    0,
+#            AuthType:      PASSWORD,
+#            DelChecks:     []DecisionCheck{},
+#            NewChecks:     []DecisionCheck{},
+#        },
+#        {
+#            TestName:      "check that 127.0.0.1 is not in deleted IP",
+#            Method:        "GET",
+#            Route:         "/v1/decisions/stream?startup=true",
+#            CheckCodeOnly: false,
+#            Code:          200,
+#            LenNew:        2,
+#            LenDeleted:    0,
+#            AuthType:      APIKEY,
+#            DelChecks:     []DecisionCheck{},
+#            NewChecks:     []DecisionCheck{},
+#        },
+#        {
+#            TestName:      "delete decisions 1 (127.0.0.1)",
+#            Method:        "DELETE",
+#            Route:         "/v1/decisions/1",
+#            CheckCodeOnly: true,
+#            Code:          200,
+#            LenNew:        0,
+#            LenDeleted:    0,
+#            AuthType:      PASSWORD,
+#            DelChecks:     []DecisionCheck{},
+#            NewChecks:     []DecisionCheck{},
+#        },
+#            TestName:      "127.0.0.1 should be in deleted now",
+#            Method:        "GET",
+#            Route:         "/v1/decisions/stream?startup=true",
+#            CheckCodeOnly: false,
+#            Code:          200,
+#            LenNew:        1,
+#            LenDeleted:    1,
+#            AuthType:      APIKEY,
+#            DelChecks: []DecisionCheck{
+#                {
+#                    ID:       int64(1),
+#                    Origin:   "test",
+#                    Scenario: "crowdsecurity/test",
+#                    Value:    "127.0.0.1",
+#                    Duration: "-", // we check that the time is negative
+#                },
+#            },
+#            NewChecks: []DecisionCheck{},
+#        },
+#}
+

tests/lib/db/instance-mysql

@@ -115,7 +115,13 @@ case "$1" in
         ;;
     exec_sql)
         shift
-        exec_sql "$@"
+        #
+        # This command is meant to run a query against the the crowdsec database.
+        # The exec_sql() function is more generic and is also used for database setup and backups.
+        #
+        # For this reason, we select the database here.
+        #
+        exec_sql "use crowdsec_test; $@"
         ;;
     *)
         about

tests/lib/setup_file.sh

@@ -4,6 +4,10 @@
 # https://github.com/bats-core/bats-core/blob/master/docs/source/warnings/BW02.rst
 bats_require_minimum_version 1.5.0
 
+# this should have effect globally, for all tests
+# https://github.com/bats-core/bats-core/blob/master/docs/source/warnings/BW02.rst
+bats_require_minimum_version 1.5.0
+
 debug() {
     echo 'exec 1<&-; exec 2<&-; exec 1>&3; exec 2>&1'
 }