Add support for bounce processing.

- Blocklist or unsubscribe subscribers based on a bounce threshold
- Add /bounces UI for viewing bounces and in the subscriber view
- Add settings UI for managing bounce settings
- Add support for scanning POP3 bounce mailboxes
- Add a generic webhook for posting custom bounces at /webhooks/bounce
- Add SES bounce webhook support at /webhooks/services/ses
- Add Sendgrid bounce webhook support at /webhooks/services/sendgrid

cmd/bounce.go

@@ -0,0 +1,251 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/knadh/listmonk/internal/subimporter"
+	"github.com/knadh/listmonk/models"
+	"github.com/labstack/echo"
+	"github.com/lib/pq"
+)
+
+type bouncesWrap struct {
+	Results []models.Bounce `json:"results"`
+
+	Total   int `json:"total"`
+	PerPage int `json:"per_page"`
+	Page    int `json:"page"`
+}
+
+// handleGetBounces handles retrieval of bounce records.
+func handleGetBounces(c echo.Context) error {
+	var (
+		app = c.Get("app").(*App)
+		pg  = getPagination(c.QueryParams(), 50)
+		out bouncesWrap
+
+		id, _     = strconv.Atoi(c.Param("id"))
+		campID, _ = strconv.Atoi(c.QueryParam("campaign_id"))
+		source    = c.FormValue("source")
+		orderBy   = c.FormValue("order_by")
+		order     = c.FormValue("order")
+	)
+
+	// Fetch one list.
+	single := false
+	if id > 0 {
+		single = true
+	}
+
+	// Sort params.
+	if !strSliceContains(orderBy, bounceQuerySortFields) {
+		orderBy = "created_at"
+	}
+	if order != sortAsc && order != sortDesc {
+		order = sortDesc
+	}
+
+	stmt := fmt.Sprintf(app.queries.QueryBounces, orderBy, order)
+	if err := db.Select(&out.Results, stmt, id, campID, 0, source, pg.Offset, pg.Limit); err != nil {
+		app.log.Printf("error fetching bounces: %v", err)
+		return echo.NewHTTPError(http.StatusInternalServerError,
+			app.i18n.Ts("globals.messages.errorFetching",
+				"name", "{globals.terms.bounce}", "error", pqErrMsg(err)))
+	}
+	if len(out.Results) == 0 {
+		out.Results = []models.Bounce{}
+		return c.JSON(http.StatusOK, okResp{out})
+	}
+
+	if single {
+		return c.JSON(http.StatusOK, okResp{out.Results[0]})
+	}
+
+	// Meta.
+	out.Total = out.Results[0].Total
+	out.Page = pg.Page
+	out.PerPage = pg.PerPage
+
+	return c.JSON(http.StatusOK, okResp{out})
+}
+
+// handleGetSubscriberBounces retrieves a subscriber's bounce records.
+func handleGetSubscriberBounces(c echo.Context) error {
+	var (
+		app   = c.Get("app").(*App)
+		subID = c.Param("id")
+	)
+
+	id, _ := strconv.ParseInt(subID, 10, 64)
+	if id < 1 {
+		return echo.NewHTTPError(http.StatusBadRequest, app.i18n.T("globals.messages.invalidID"))
+	}
+
+	out := []models.Bounce{}
+	stmt := fmt.Sprintf(app.queries.QueryBounces, "created_at", "ASC")
+	if err := db.Select(&out, stmt, 0, 0, subID, "", 0, 1000); err != nil {
+		app.log.Printf("error fetching bounces: %v", err)
+		return echo.NewHTTPError(http.StatusInternalServerError,
+			app.i18n.Ts("globals.messages.errorFetching",
+				"name", "{globals.terms.bounce}", "error", pqErrMsg(err)))
+	}
+
+	return c.JSON(http.StatusOK, okResp{out})
+}
+
+// handleDeleteBounces handles bounce deletion, either a single one (ID in the URI), or a list.
+func handleDeleteBounces(c echo.Context) error {
+	var (
+		app    = c.Get("app").(*App)
+		pID    = c.Param("id")
+		all, _ = strconv.ParseBool(c.QueryParam("all"))
+		IDs    = pq.Int64Array{}
+	)
+
+	// Is it an /:id call?
+	if pID != "" {
+		id, _ := strconv.ParseInt(pID, 10, 64)
+		if id < 1 {
+			return echo.NewHTTPError(http.StatusBadRequest, app.i18n.T("globals.messages.invalidID"))
+		}
+		IDs = append(IDs, id)
+	} else if !all {
+		// Multiple IDs.
+		i, err := parseStringIDs(c.Request().URL.Query()["id"])
+		if err != nil {
+			return echo.NewHTTPError(http.StatusBadRequest,
+				app.i18n.Ts("globals.messages.invalidID", "error", err.Error()))
+		}
+
+		if len(i) == 0 {
+			return echo.NewHTTPError(http.StatusBadRequest,
+				app.i18n.Ts("globals.messages.invalidID"))
+		}
+		IDs = i
+	}
+
+	if _, err := app.queries.DeleteBounces.Exec(IDs); err != nil {
+		app.log.Printf("error deleting bounces: %v", err)
+		return echo.NewHTTPError(http.StatusInternalServerError,
+			app.i18n.Ts("globals.messages.errorDeleting",
+				"name", "{globals.terms.bounce}", "error", pqErrMsg(err)))
+	}
+
+	return c.JSON(http.StatusOK, okResp{true})
+}
+
+// handleBounceWebhook renders the HTML preview of a template.
+func handleBounceWebhook(c echo.Context) error {
+	var (
+		app     = c.Get("app").(*App)
+		service = c.Param("service")
+
+		bounces []models.Bounce
+	)
+
+	// Read the request body instead of using using c.Bind() to read to save the entire raw request as meta.
+	rawReq, err := ioutil.ReadAll(c.Request().Body)
+	if err != nil {
+		app.log.Printf("error reading ses notification body: %v", err)
+		return echo.NewHTTPError(http.StatusBadRequest, app.i18n.Ts("globals.messages.internalError"))
+	}
+
+	switch true {
+	// Native internal webhook.
+	case service == "":
+		var b models.Bounce
+		if err := json.Unmarshal(rawReq, &b); err != nil {
+			return echo.NewHTTPError(http.StatusBadRequest, app.i18n.Ts("globals.messages.invalidData"))
+		}
+
+		if err := validateBounceFields(b, app); err != nil {
+			return err
+		}
+
+		b.Email = strings.ToLower(b.Email)
+
+		if len(b.Meta) == 0 {
+			b.Meta = json.RawMessage("{}")
+		}
+
+		if b.CreatedAt.Year() == 0 {
+			b.CreatedAt = time.Now()
+		}
+
+		bounces = append(bounces, b)
+
+	// Amazon SES.
+	case service == "ses" && app.constants.BounceSESEnabled:
+		switch c.Request().Header.Get("X-Amz-Sns-Message-Type") {
+		// SNS webhook registration confirmation. Only after these are processed will the endpoint
+		// start getting bounce notifications.
+		case "SubscriptionConfirmation", "UnsubscribeConfirmation":
+			if err := app.bounce.SES.ProcessSubscription(rawReq); err != nil {
+				app.log.Printf("error processing SNS (SES) subscription: %v", err)
+				return echo.NewHTTPError(http.StatusBadRequest, app.i18n.T("globals.messages.invalidData"))
+			}
+			break
+
+		// Bounce notification.
+		case "Notification":
+			b, err := app.bounce.SES.ProcessBounce(rawReq)
+			if err != nil {
+				app.log.Printf("error processing SES notification: %v", err)
+				return echo.NewHTTPError(http.StatusBadRequest, app.i18n.T("globals.messages.invalidData"))
+			}
+			bounces = append(bounces, b)
+
+		default:
+			return echo.NewHTTPError(http.StatusBadRequest, app.i18n.T("globals.messages.invalidData"))
+		}
+
+	// SendGrid.
+	case service == "sendgrid" && app.constants.BounceSendgridEnabled:
+		var (
+			sig = c.Request().Header.Get("X-Twilio-Email-Event-Webhook-Signature")
+			ts  = c.Request().Header.Get("X-Twilio-Email-Event-Webhook-Timestamp")
+		)
+
+		// Sendgrid sends multiple bounces.
+		bs, err := app.bounce.Sendgrid.ProcessBounce(sig, ts, rawReq)
+		if err != nil {
+			app.log.Printf("error processing sendgrid notification: %v", err)
+			return echo.NewHTTPError(http.StatusBadRequest, app.i18n.T("globals.messages.invalidData"))
+		}
+		bounces = append(bounces, bs...)
+
+	default:
+		return echo.NewHTTPError(http.StatusBadRequest, app.i18n.Ts("bounces.unknownService"))
+	}
+
+	// Record bounces if any.
+	for _, b := range bounces {
+		if err := app.bounce.Record(b); err != nil {
+			app.log.Printf("error recording bounce: %v", err)
+		}
+	}
+
+	return c.JSON(http.StatusOK, okResp{true})
+}
+
+func validateBounceFields(b models.Bounce, app *App) error {
+	if b.Email == "" && b.SubscriberUUID == "" {
+		return echo.NewHTTPError(http.StatusBadRequest, app.i18n.T("globals.messages.invalidData"))
+	}
+
+	if b.Email != "" && !subimporter.IsEmail(b.Email) {
+		return echo.NewHTTPError(http.StatusBadRequest, app.i18n.T("globals.messages.invalidEmail"))
+	}
+
+	if b.SubscriberUUID != "" && !reUUID.MatchString(b.SubscriberUUID) {
+		return echo.NewHTTPError(http.StatusBadRequest, app.i18n.T("globals.messages.invalidUUID"))
+	}
+
+	return nil
+}

cmd/campaigns.go

@@ -73,6 +73,7 @@ var (
 	regexFullTextQuery = regexp.MustCompile(`\s+`)
 
 	campaignQuerySortFields = []string{"name", "status", "created_at", "updated_at"}
+	bounceQuerySortFields   = []string{"email", "campaign_name", "source", "created_at"}
 )
 
 // handleGetCampaigns handles retrieval of campaigns.

cmd/handlers.go

@@ -62,6 +62,8 @@ func registerHTTPHandlers(e *echo.Echo, app *App) {
 
 	g.GET("/api/subscribers/:id", handleGetSubscriber)
 	g.GET("/api/subscribers/:id/export", handleExportSubscriberData)
+	g.GET("/api/subscribers/:id/bounces", handleGetSubscriberBounces)
+	g.DELETE("/api/subscribers/:id/bounces", handleDeleteSubscriberBounces)
 	g.POST("/api/subscribers", handleCreateSubscriber)
 	g.PUT("/api/subscribers/:id", handleUpdateSubscriber)
 	g.POST("/api/subscribers/:id/optin", handleSubscriberSendOptin)
@@ -72,6 +74,10 @@ func registerHTTPHandlers(e *echo.Echo, app *App) {
 	g.DELETE("/api/subscribers/:id", handleDeleteSubscribers)
 	g.DELETE("/api/subscribers", handleDeleteSubscribers)
 
+	g.GET("/api/bounces", handleGetBounces)
+	g.DELETE("/api/bounces", handleDeleteBounces)
+	g.DELETE("/api/bounces/:id", handleDeleteBounces)
+
 	// Subscriber operations based on arbitrary SQL queries.
 	// These aren't very REST-like.
 	g.POST("/api/subscribers/query/delete", handleDeleteSubscribersByQuery)
@@ -132,6 +138,14 @@ func registerHTTPHandlers(e *echo.Echo, app *App) {
 	g.GET("/settings", handleIndexPage)
 	g.GET("/settings/logs", handleIndexPage)
 
+	if app.constants.BounceWebhooksEnabled {
+		// Private authenticated bounce endpoint.
+		g.POST("/webhooks/bounce", handleBounceWebhook)
+
+		// Public bounce endpoints for webservices like SES.
+		e.POST("/webhooks/service/:service", handleBounceWebhook)
+	}
+
 	// Public subscriber facing views.
 	e.GET("/subscription/form", handleSubscriptionFormPage)
 	e.POST("/subscription/form", handleSubscriptionForm)

cmd/init.go

@@ -21,6 +21,8 @@ import (
 	"github.com/knadh/koanf/providers/confmap"
 	"github.com/knadh/koanf/providers/file"
 	"github.com/knadh/koanf/providers/posflag"
+	"github.com/knadh/listmonk/internal/bounce"
+	"github.com/knadh/listmonk/internal/bounce/mailbox"
 	"github.com/knadh/listmonk/internal/i18n"
 	"github.com/knadh/listmonk/internal/manager"
 	"github.com/knadh/listmonk/internal/media"
@@ -65,6 +67,10 @@ type constants struct {
 	OptinURL      string
 	MessageURL    string
 	MediaProvider string
+
+	BounceWebhooksEnabled bool
+	BounceSESEnabled      bool
+	BounceSendgridEnabled bool
 }
 
 func initFlags() {
@@ -296,6 +302,10 @@ func initConstants() *constants {
 
 	// url.com/campaign/{campaign_uuid}/{subscriber_uuid}/px.png
 	c.ViewTrackURL = fmt.Sprintf("%s/campaign/%%s/%%s/px.png", c.RootURL)
+
+	c.BounceWebhooksEnabled = ko.Bool("bounce.webhooks_enabled")
+	c.BounceSESEnabled = ko.Bool("bounce.ses_enabled")
+	c.BounceSendgridEnabled = ko.Bool("bounce.sendgrid_enabled")
 	return &c
 }
 
@@ -344,8 +354,7 @@ func initCampaignManager(q *Queries, cs *constants, app *App) *manager.Manager {
 		SlidingWindow:         ko.Bool("app.message_sliding_window"),
 		SlidingWindowDuration: ko.Duration("app.message_sliding_window_duration"),
 		SlidingWindowRate:     ko.Int("app.message_sliding_window_rate"),
-	}, newManagerDB(q), campNotifCB, app.i18n, lo)
+	}, newManagerStore(q), campNotifCB, app.i18n, lo)
-
 }
 
 // initImporter initializes the bulk subscriber importer.
@@ -495,6 +504,45 @@ func initNotifTemplates(path string, fs stuffbin.FileSystem, i *i18n.I18n, cs *c
 	return tpl
 }
 
+// initBounceManager initializes the bounce manager that scans mailboxes and listens to webhooks
+// for incoming bounce events.
+func initBounceManager(app *App) *bounce.Manager {
+	opt := bounce.Opt{
+		BounceCount:     ko.MustInt("bounce.count"),
+		BounceAction:    ko.MustString("bounce.action"),
+		WebhooksEnabled: ko.Bool("bounce.webhooks_enabled"),
+		SESEnabled:      ko.Bool("bounce.ses_enabled"),
+		SendgridEnabled: ko.Bool("bounce.sendgrid_enabled"),
+		SendgridKey:     ko.String("bounce.sendgrid_key"),
+	}
+
+	// For now, only one mailbox is supported.
+	for _, b := range ko.Slices("bounce.mailboxes") {
+		if !b.Bool("enabled") {
+			continue
+		}
+
+		var boxOpt mailbox.Opt
+		if err := b.UnmarshalWithConf("", &boxOpt, koanf.UnmarshalConf{Tag: "json"}); err != nil {
+			lo.Fatalf("error reading bounce mailbox config: %v", err)
+		}
+
+		opt.MailboxType = b.String("type")
+		opt.MailboxEnabled = true
+		opt.Mailbox = boxOpt
+		break
+	}
+
+	b, err := bounce.New(opt, &bounce.Queries{
+		RecordQuery: app.queries.RecordBounce,
+	}, app.log)
+	if err != nil {
+		lo.Fatalf("error initializing bounce manager: %v", err)
+	}
+
+	return b
+}
+
 // initHTTPServer sets up and runs the app's main HTTP server and blocks forever.
 func initHTTPServer(app *App) *echo.Echo {
 	// Initialize the HTTP server.

cmd/lists.go

@@ -160,8 +160,7 @@ func handleUpdateList(c echo.Context) error {
 	return handleGetLists(c)
 }
 
-// handleDeleteLists handles deletion deletion,
+// handleDeleteLists handles list deletion, either a single one (ID in the URI), or a list.
-// either a single one (ID in the URI), or a list.
 func handleDeleteLists(c echo.Context) error {
 	var (
 		app   = c.Get("app").(*App)

cmd/main.go

@@ -17,6 +17,7 @@ import (
 	"github.com/jmoiron/sqlx"
 	"github.com/knadh/koanf"
 	"github.com/knadh/koanf/providers/env"
+	"github.com/knadh/listmonk/internal/bounce"
 	"github.com/knadh/listmonk/internal/buflog"
 	"github.com/knadh/listmonk/internal/i18n"
 	"github.com/knadh/listmonk/internal/manager"
@@ -42,6 +43,7 @@ type App struct {
 	messengers map[string]messenger.Messenger
 	media      media.Store
 	i18n       *i18n.I18n
+	bounce     *bounce.Manager
 	notifTpls  *template.Template
 	log        *log.Logger
 	bufLog     *buflog.BufLog
@@ -168,6 +170,11 @@ func main() {
 	app.importer = initImporter(app.queries, db, app)
 	app.notifTpls = initNotifTemplates("/email-templates/*.html", fs, app.i18n, app.constants)
 
+	if ko.Bool("bounce.enabled") {
+		app.bounce = initBounceManager(app)
+		go app.bounce.Run()
+	}
+
 	// Initialize the default SMTP (`email`) messenger.
 	app.messengers[emailMsgr] = initSMTPMessenger(app.manager)
 

cmd/manager_store.go

@@ -12,7 +12,7 @@ type runnerDB struct {
 	queries *Queries
 }
 
-func newManagerDB(q *Queries) *runnerDB {
+func newManagerStore(q *Queries) *runnerDB {
 	return &runnerDB{
 		queries: q,
 	}
@@ -64,3 +64,31 @@ func (r *runnerDB) CreateLink(url string) (string, error) {
 
 	return out, nil
 }
+
+// RecordBounce records a bounce event and returns the bounce count.
+func (r *runnerDB) RecordBounce(b models.Bounce) (int64, int, error) {
+	var res = struct {
+		SubscriberID int64 `db:"subscriber_id"`
+		Num          int   `db:"num"`
+	}{}
+
+	err := r.queries.UpdateCampaignStatus.Select(&res,
+		b.SubscriberUUID,
+		b.Email,
+		b.CampaignUUID,
+		b.Type,
+		b.Source,
+		b.Meta)
+
+	return res.SubscriberID, res.Num, err
+}
+
+func (r *runnerDB) BlocklistSubscriber(id int64) error {
+	_, err := r.queries.BlocklistSubscribers.Exec(pq.Int64Array{id})
+	return err
+}
+
+func (r *runnerDB) DeleteSubscriber(id int64) error {
+	_, err := r.queries.DeleteSubscribers.Exec(pq.Int64Array{id})
+	return err
+}

cmd/queries.go

@@ -83,6 +83,10 @@ type Queries struct {
 	UpdateSettings *sqlx.Stmt `query:"update-settings"`
 
 	// GetStats *sqlx.Stmt `query:"get-stats"`
+	RecordBounce              *sqlx.Stmt `query:"record-bounce"`
+	QueryBounces              string     `query:"query-bounces"`
+	DeleteBounces             *sqlx.Stmt `query:"delete-bounces"`
+	DeleteBouncesBySubscriber *sqlx.Stmt `query:"delete-bounces-by-subscriber"`
 }
 
 // dbConf contains database config required for connecting to a DB.

cmd/settings.go

@@ -80,6 +80,28 @@ type settings struct {
 		Timeout       string `json:"timeout"`
 		MaxMsgRetries int    `json:"max_msg_retries"`
 	} `json:"messengers"`
+
+	BounceEnabled        bool   `json:"bounce.enabled"`
+	BounceEnableWebhooks bool   `json:"bounce.webhooks_enabled"`
+	BounceCount          int    `json:"bounce.count"`
+	BounceAction         string `json:"bounce.action"`
+	SESEnabled           bool   `json:"bounce.ses_enabled"`
+	SendgridEnabled      bool   `json:"bounce.sendgrid_enabled"`
+	SendgridKey          string `json:"bounce.sendgrid_key"`
+	BounceBoxes          []struct {
+		UUID          string `json:"uuid"`
+		Enabled       bool   `json:"enabled"`
+		Type          string `json:"type"`
+		Host          string `json:"host"`
+		Port          int    `json:"port"`
+		AuthProtocol  string `json:"auth_protocol"`
+		ReturnPath    string `json:"return_path"`
+		Username      string `json:"username"`
+		Password      string `json:"password,omitempty"`
+		TLSEnabled    bool   `json:"tls_enabled"`
+		TLSSkipVerify bool   `json:"tls_skip_verify"`
+		ScanInterval  string `json:"scan_interval"`
+	} `json:"bounce.mailboxes"`
 }
 
 var (
@@ -99,10 +121,14 @@ func handleGetSettings(c echo.Context) error {
 	for i := 0; i < len(s.SMTP); i++ {
 		s.SMTP[i].Password = ""
 	}
+	for i := 0; i < len(s.BounceBoxes); i++ {
+		s.BounceBoxes[i].Password = ""
+	}
 	for i := 0; i < len(s.Messengers); i++ {
 		s.Messengers[i].Password = ""
 	}
 	s.UploadS3AwsSecretAccessKey = ""
+	s.SendgridKey = ""
 
 	return c.JSON(http.StatusOK, okResp{s})
 }
@@ -154,6 +180,31 @@ func handleUpdateSettings(c echo.Context) error {
 		return echo.NewHTTPError(http.StatusBadRequest, app.i18n.T("settings.errorNoSMTP"))
 	}
 
+	// Bounce boxes.
+	for i, s := range set.BounceBoxes {
+		// Assign a UUID. The frontend only sends a password when the user explictly
+		// changes the password. In other cases, the existing password in the DB
+		// is copied while updating the settings and the UUID is used to match
+		// the incoming array of blocks with the array in the DB.
+		if s.UUID == "" {
+			set.BounceBoxes[i].UUID = uuid.Must(uuid.NewV4()).String()
+		}
+
+		if d, _ := time.ParseDuration(s.ScanInterval); d.Minutes() < 1 {
+			return echo.NewHTTPError(http.StatusBadRequest, app.i18n.T("settings.bounces.invalidScanInterval"))
+		}
+
+		// If there's no password coming in from the frontend, copy the existing
+		// password by matching the UUID.
+		if s.Password == "" {
+			for _, c := range cur.BounceBoxes {
+				if s.UUID == c.UUID {
+					set.BounceBoxes[i].Password = c.Password
+				}
+			}
+		}
+	}
+
 	// Validate and sanitize postback Messenger names. Duplicates are disallowed
 	// and "email" is a reserved name.
 	names := map[string]bool{emailMsgr: true}
@@ -189,6 +240,9 @@ func handleUpdateSettings(c echo.Context) error {
 	if set.UploadS3AwsSecretAccessKey == "" {
 		set.UploadS3AwsSecretAccessKey = cur.UploadS3AwsSecretAccessKey
 	}
+	if set.SendgridKey == "" {
+		set.SendgridKey = cur.SendgridKey
+	}
 
 	// Marshal settings.
 	b, err := json.Marshal(set)

cmd/subscribers.go

@@ -614,6 +614,28 @@ func handleManageSubscriberListsByQuery(c echo.Context) error {
 	return c.JSON(http.StatusOK, okResp{true})
 }
 
+// handleDeleteSubscriberBounces deletes all the bounces on a subscriber.
+func handleDeleteSubscriberBounces(c echo.Context) error {
+	var (
+		app = c.Get("app").(*App)
+		pID = c.Param("id")
+	)
+
+	id, _ := strconv.ParseInt(pID, 10, 64)
+	if id < 1 {
+		return echo.NewHTTPError(http.StatusBadRequest, app.i18n.T("globals.messages.invalidID"))
+	}
+
+	if _, err := app.queries.DeleteBouncesBySubscriber.Exec(id, nil); err != nil {
+		app.log.Printf("error deleting bounces: %v", err)
+		return echo.NewHTTPError(http.StatusInternalServerError,
+			app.i18n.Ts("globals.messages.errorDeleting",
+				"name", "{globals.terms.bounces}", "error", pqErrMsg(err)))
+	}
+
+	return c.JSON(http.StatusOK, okResp{true})
+}
+
 // handleExportSubscriberData pulls the subscriber's profile,
 // list subscriptions, campaign views and clicks and produces
 // a JSON report. This is a privacy feature and depends on the

cmd/upgrade.go

@@ -30,6 +30,7 @@ var migList = []migFunc{
 	{"v0.8.0", migrations.V0_8_0},
 	{"v0.9.0", migrations.V0_9_0},
 	{"v1.0.0", migrations.V1_0_0},
+	{"v2.0.0", migrations.V2_0_0},
 }
 
 // upgrade upgrades the database to the current version by running SQL migration files

frontend/fontello/config.json

@@ -510,6 +510,34 @@
         "magnify"
       ]
     },
+    {
+      "uid": "e97fad4c93444c9b81151c2aa4086e13",
+      "css": "chart-bar",
+      "code": 59428,
+      "src": "custom_icons",
+      "selected": true,
+      "svg": {
+        "path": "M916 875H84V125H166V791H250V416H416V791H500V250H666V791H750V584H916V875Z",
+        "width": 1000
+      },
+      "search": [
+        "chart-bar"
+      ]
+    },
+    {
+      "uid": "61e03b48670cd93477e233e0d6bb3f1c",
+      "css": "email-bounce",
+      "code": 59429,
+      "src": "custom_icons",
+      "selected": true,
+      "svg": {
+        "path": "M916 834H750V959L541 771.5 750 584V709H916V834ZM834 166H166Q132.8 166 108.4 190.4T84 250V750Q84 785.2 108.4 809.6T166 834H459V750H166V334L500 541 834 334V625H916V250Q916 214.8 891.6 190.4T834 166ZM500 459L166 250H834Z",
+        "width": 1000
+      },
+      "search": [
+        "email-bounce"
+      ]
+    },
     {
       "uid": "f4ad3f6d071a0bfb3a8452b514ed0892",
       "css": "vector-square",
@@ -4570,20 +4598,6 @@
         "chart-areaspline"
       ]
     },
-    {
-      "uid": "e97fad4c93444c9b81151c2aa4086e13",
-      "css": "chart-bar",
-      "code": 983336,
-      "src": "custom_icons",
-      "selected": false,
-      "svg": {
-        "path": "M916 875H84V125H166V791H250V416H416V791H500V250H666V791H750V584H916V875Z",
-        "width": 1000
-      },
-      "search": [
-        "chart-bar"
-      ]
-    },
     {
       "uid": "298bc9b464d2b4e5cad91cd3d419747f",
       "css": "chart-histogram",
@@ -60444,20 +60458,6 @@
         "email-receive"
       ]
     },
-    {
-      "uid": "61e03b48670cd93477e233e0d6bb3f1c",
-      "css": "email-receive-outline",
-      "code": 987355,
-      "src": "custom_icons",
-      "selected": false,
-      "svg": {
-        "path": "M916 834H750V959L541 771.5 750 584V709H916V834ZM834 166H166Q132.8 166 108.4 190.4T84 250V750Q84 785.2 108.4 809.6T166 834H459V750H166V334L500 541 834 334V625H916V250Q916 214.8 891.6 190.4T834 166ZM500 459L166 250H834Z",
-        "width": 1000
-      },
-      "search": [
-        "email-receive-outline"
-      ]
-    },
     {
       "uid": "b52d1e4a907f8f98e038e8997079e456",
       "css": "email-send",

frontend/src/App.vue

@@ -55,6 +55,10 @@
                   <b-menu-item :to="{name: 'import'}" tag="router-link"
                     :active="activeItem.import" data-cy="import"
                     icon="file-upload-outline" :label="$t('menu.import')"></b-menu-item>
+
+                  <b-menu-item :to="{name: 'bounces'}" tag="router-link"
+                    :active="activeItem.bounces" data-cy="bounces"
+                    icon="email-bounce" :label="$t('globals.terms.bounces')"></b-menu-item>
                 </b-menu-item><!-- subscribers -->
 
                 <b-menu-item :expanded="activeGroup.campaigns"

frontend/src/api/index.js

@@ -111,6 +111,21 @@ export const deleteList = (id) => http.delete(`/api/lists/${id}`,
 export const getSubscribers = async (params) => http.get('/api/subscribers',
   { params, loading: models.subscribers, store: models.subscribers });
 
+export const getSubscriber = async (id) => http.get(`/api/subscribers/${id}`,
+  { loading: models.subscribers });
+
+export const getSubscriberBounces = async (id) => http.get(`/api/subscribers/${id}/bounces`,
+  { loading: models.bounces });
+
+export const deleteSubscriberBounces = async (id) => http.delete(`/api/subscribers/${id}/bounces`,
+  { loading: models.bounces });
+
+export const deleteBounce = async (id) => http.delete(`/api/bounces/${id}`,
+  { loading: models.bounces });
+
+export const deleteBounces = async (params) => http.delete('/api/bounces',
+  { params, loading: models.bounces });
+
 export const createSubscriber = (data) => http.post('/api/subscribers', data,
   { loading: models.subscribers });
 
@@ -148,6 +163,10 @@ export const getImportLogs = async () => http.get('/api/import/subscribers/logs'
 
 export const stopImport = () => http.delete('/api/import/subscribers');
 
+// Bounces.
+export const getBounces = async (params) => http.get('/api/bounces',
+  { params, loading: models.bounces });
+
 // Campaigns.
 export const getCampaigns = async (params) => http.get('/api/campaigns',
   { params, loading: models.campaigns, store: models.campaigns });

frontend/src/assets/icons/fontello.css

@@ -5,37 +5,37 @@
   font-style: normal;
 }
  
- [class^="mdi-"]:before, [class*=" mdi-"]:before {
+[class^="mdi-"]:before, [class*=" mdi-"]:before {
   font-family: "fontello";
   font-style: normal;
   font-weight: normal;
   speak: never;
- 
+
   display: inline-block;
   text-decoration: inherit;
   width: 1em;
   margin-right: .2em;
   text-align: center;
   /* opacity: .8; */
- 
+
   /* For safety - reset parent styles, that can break glyph codes*/
   font-variant: normal;
   text-transform: none;
- 
+
   /* fix buttons height, for twitter bootstrap */
   line-height: 1em;
- 
+
   /* Animation center compensation - margins should be symmetric */
   /* remove if not needed */
   margin-left: .2em;
- 
+
   /* you can be more comfortable with increased icons size */
   /* font-size: 120%; */
- 
+
   /* Font smoothing. That was taken from TWBS */
   -webkit-font-smoothing: antialiased;
   -moz-osx-font-smoothing: grayscale;
- 
+
   /* Uncomment for 3D effect */
   /* text-shadow: 1px 1px 1px rgba(127, 127, 127, 0.3); */
 }
@@ -76,3 +76,5 @@
 .mdi-arrow-down:before { content: '\e821'; } /* '' */
 .mdi-cancel:before { content: '\e822'; } /* '' */
 .mdi-magnify:before { content: '\e823'; } /* '' */
+.mdi-chart-bar:before { content: '\e824'; } /* '' */
+.mdi-email-bounce:before { content: '\e825'; } /* '' */

frontend/src/assets/style.scss

@@ -92,7 +92,17 @@ section {
 }
 
 .box {
-  box-shadow: 0 0 2px $grey-lighter;
+  background: $white-bis;
+  box-shadow: 2px 2px 5px $white-ter;
+  border: 1px solid $grey-lightest;
+
+  hr {
+    background-color: #efefef;
+  }
+}
+
+.page-header {
+  min-height: 60px;
 }
 
 /* Two column sidebar+body layout */
@@ -323,7 +333,7 @@ section {
 
 /* Tabs */
 .b-tabs .tab-content {
-  padding-top: 2rem;
+  padding-top: 3rem;
 }
 
 /* Tags */
@@ -449,6 +459,12 @@ section.lists {
   }
 }
 
+.bounces {
+  pre {
+    padding: 5px 10px;
+  }
+}
+
 /* Import page */
 section.import {
   .delimiter input {

frontend/src/constants.js

@@ -7,6 +7,7 @@ export const models = Object.freeze({
   campaigns: 'campaigns',
   templates: 'templates',
   media: 'media',
+  bounces: 'bounces',
   settings: 'settings',
   logs: 'logs',
 });

frontend/src/router/index.js

@@ -35,6 +35,12 @@ const routes = [
     meta: { title: 'Import subscribers', group: 'subscribers' },
     component: () => import(/* webpackChunkName: "main" */ '../views/Import.vue'),
   },
+  {
+    path: '/subscribers/bounces',
+    name: 'bounces',
+    meta: { title: 'Bounces', group: 'subscribers' },
+    component: () => import(/* webpackChunkName: "main" */ '../views/Bounces.vue'),
+  },
   {
     path: '/subscribers/lists/:listID',
     name: 'subscribers_list',

frontend/src/views/Bounces.vue

@@ -0,0 +1,196 @@
+<template>
+  <section class="bounces">
+    <header class="page-header columns">
+      <div class="column is-two-thirds">
+        <h1 class="title is-4">{{ $t('globals.terms.bounces') }}
+          <span v-if="bounces.total > 0">({{ bounces.total }})</span></h1>
+      </div>
+      <div class="column has-text-right buttons">
+        <b-button v-if="bulk.checked.length > 0 || bulk.all" type="is-primary"
+          icon-left="trash-can-outline" data-cy="btn-delete"
+          @click.prevent="$utils.confirm(null, () => deleteBounces())">
+          {{ $t('globals.buttons.delete') }}
+        </b-button>
+        <b-button v-if="bounces.total" icon-left="trash-can-outline" data-cy="btn-delete"
+          @click.prevent="$utils.confirm(null, () => deleteBounces(true))">
+          {{ $t('globals.buttons.deleteAll') }}
+        </b-button>
+      </div>
+    </header>
+
+    <b-table :data="bounces.results" :hoverable="true" :loading="loading.bounces"
+      default-sort="createdAt"
+      checkable
+      @check-all="onTableCheck" @check="onTableCheck"
+      :checked-rows.sync="bulk.checked"
+      detailed
+      show-detail-icon
+      @details-open="(row) => $buefy.toast.open(`Expanded ${row.user.first_name}`)"
+      paginated backend-pagination pagination-position="both" @page-change="onPageChange"
+      :current-page="queryParams.page" :per-page="bounces.perPage" :total="bounces.total"
+      backend-sorting @sort="onSort">
+      <b-table-column v-slot="props" field="email" :label="$t('subscribers.email')"
+        :td-attrs="$utils.tdID" sortable>
+        <router-link :to="{ name: 'subscriber', params: { id: props.row.subscriberId }}">
+          {{ props.row.email }}
+        </router-link>
+      </b-table-column>
+
+      <b-table-column v-slot="props" field="campaign_name" :label="$tc('globals.terms.campaign')"
+        sortable>
+        <router-link :to="{ name: 'bounces', query: { campaign_id: props.row.campaign.id }}">
+          {{ props.row.campaign.name }}
+        </router-link>
+      </b-table-column>
+
+      <b-table-column v-slot="props" field="source" :label="$t('bounces.source')" sortable>
+        <router-link :to="{ name: 'bounces', query: { source: props.row.source } }">
+          {{ props.row.source }}
+        </router-link>
+      </b-table-column>
+
+      <b-table-column v-slot="props" field="created_at"
+        :label="$t('globals.fields.createdAt')" sortable>
+        {{ $utils.niceDate(props.row.createdAt, true) }}
+      </b-table-column>
+
+      <b-table-column v-slot="props" cell-class="actions" align="right">
+        <div>
+          <a v-if="!props.row.isDefault" href="#"
+            @click.prevent="$utils.confirm(null, () => deleteBounce(props.row))"
+            data-cy="btn-delete">
+            <b-tooltip :label="$t('globals.buttons.delete')" type="is-dark">
+              <b-icon icon="trash-can-outline" size="is-small" />
+            </b-tooltip>
+          </a>
+          <span v-else class="a has-text-grey-light">
+              <b-icon icon="trash-can-outline" size="is-small" />
+          </span>
+        </div>
+      </b-table-column>
+
+      <template #detail="props">
+        <pre class="is-size-7">{{ props.row.meta }}</pre>
+      </template>
+
+      <template #empty v-if="!loading.templates">
+        <empty-placeholder />
+      </template>
+    </b-table>
+  </section>
+</template>
+
+<script>
+import Vue from 'vue';
+import { mapState } from 'vuex';
+import EmptyPlaceholder from '../components/EmptyPlaceholder.vue';
+
+export default Vue.extend({
+  components: {
+    EmptyPlaceholder,
+  },
+
+  data() {
+    return {
+      bounces: {},
+
+      // Table bulk row selection states.
+      bulk: {
+        checked: [],
+        all: false,
+      },
+
+      // Query params to filter the getSubscribers() API call.
+      queryParams: {
+        page: 1,
+        orderBy: 'created_at',
+        order: 'desc',
+        campaignID: 0,
+        source: '',
+      },
+    };
+  },
+
+  methods: {
+    onSort(field, direction) {
+      this.queryParams.orderBy = field;
+      this.queryParams.order = direction;
+      this.getBounces();
+    },
+
+    onPageChange(p) {
+      this.queryParams.page = p;
+      this.getBounces();
+    },
+
+    onTableCheck() {
+      // Disable bulk.all selection if there are no rows checked in the table.
+      if (this.bulk.checked.length !== this.bounces.total) {
+        this.bulk.all = false;
+      }
+    },
+
+    getBounces() {
+      this.bulk.checked = [];
+      this.bulk.all = false;
+
+      this.$api.getBounces({
+        page: this.queryParams.page,
+        order_by: this.queryParams.orderBy,
+        order: this.queryParams.order,
+        campaign_id: this.queryParams.campaign_id,
+        source: this.queryParams.source,
+      }).then((data) => {
+        this.bounces = data;
+      });
+    },
+
+    deleteBounce(b) {
+      this.$api.deleteBounce(b.id).then(() => {
+        this.getBounces();
+        this.$utils.toast(this.$t('globals.messages.deleted', { name: b.email }));
+      });
+    },
+
+    deleteBounces(all) {
+      const fnSuccess = () => {
+        this.getBounces();
+        this.$utils.toast(this.$t('globals.messages.deletedCount',
+          { name: this.$tc('globals.terms.bounces'), num: this.bounces.total }));
+      };
+
+      if (all) {
+        this.$api.deleteBounces({ all: true }).then(fnSuccess);
+        return;
+      }
+
+      const ids = this.bulk.checked.map((s) => s.id);
+      this.$api.deleteBounces({ id: ids }).then(fnSuccess);
+    },
+  },
+
+  computed: {
+    ...mapState(['templates', 'loading']),
+
+    selectedBounces() {
+      if (this.bulk.all) {
+        return this.bounces.total;
+      }
+      return this.bulk.checked.length;
+    },
+
+  },
+
+  mounted() {
+    if (this.$route.query.campaign_id) {
+      this.queryParams.campaign_id = parseInt(this.$route.query.campaign_id, 10);
+    }
+
+    if (this.$route.query.source) {
+      this.queryParams.source = this.$route.query.source;
+    }
+
+    this.getBounces();
+  },
+});
+</script>

frontend/src/views/Campaigns.vue

@@ -117,6 +117,12 @@
             <label>{{ $t('campaigns.sent') }}</label>
             {{ stats.sent }} / {{ stats.toSend }}
           </p>
+          <p>
+            <label>{{ $t('globals.terms.bounces') }}</label>
+            <router-link :to="{name: 'bounces', query: { campaign_id: props.row.id }}">
+              {{ props.row.bounces }}
+            </router-link>
+          </p>
           <p title="Speed" v-if="stats.rate">
             <label><b-icon icon="speedometer" size="is-small"></b-icon></label>
             <span class="send-rate">

frontend/src/views/Settings.vue

@@ -433,6 +433,173 @@
             </b-button>
           </b-tab-item><!-- mail servers -->
 
+          <b-tab-item :label="$t('settings.bounces.name')">
+            <div class="columns mb-6">
+              <div class="column">
+                <b-field :label="$t('settings.bounces.enable')">
+                  <b-switch v-model="form['bounce.enabled']" name="bounce.enabled" />
+                </b-field>
+              </div>
+              <div class="column" :class="{'disabled': !form['bounce.enabled']}">
+                <b-field :label="$t('settings.bounces.count')" label-position="on-border"
+                  :message="$t('settings.bounces.countHelp')">
+                  <b-numberinput v-model="form['bounce.count']"
+                    name="bounce.count" type="is-light"
+                    controls-position="compact" placeholder="3" min="1" max="1000" />
+                </b-field>
+              </div>
+              <div class="column" :class="{'disabled': !form['bounce.enabled']}">
+                <b-field :label="$t('settings.bounces.action')" label-position="on-border">
+                  <b-select name="bounce.action" v-model="form['bounce.action']">
+                    <option value="blocklist">{{ $t('settings.bounces.blocklist') }}</option>
+                    <option value="delete">{{ $t('settings.bounces.delete') }}</option>
+                  </b-select>
+                </b-field>
+              </div>
+            </div><!-- columns -->
+
+            <div class="mb-6">
+              <b-field :label="$t('settings.bounces.enableWebhooks')">
+                <b-switch v-model="form['bounce.webhooks_enabled']"
+                  :disabled="!form['bounce.enabled']"
+                  name="webhooks_enabled" :native-value="true"
+                  data-cy="btn-enable-bounce-webhook" />
+                <p class="has-text-grey">
+                  <a href="" target="_blank">{{ $t('globals.buttons.learnMore') }} &rarr;</a>
+                </p>
+              </b-field>
+              <div class="box" v-if="form['bounce.webhooks_enabled']">
+                  <div class="columns">
+                    <div class="column">
+                      <b-field :label="$t('settings.bounces.enableSES')">
+                        <b-switch v-model="form['bounce.ses_enabled']"
+                          name="ses_enabled" :native-value="true" data-cy="btn-enable-bounce-ses" />
+                      </b-field>
+                    </div>
+                  </div>
+                  <div class="columns">
+                    <div class="column is-3">
+                      <b-field :label="$t('settings.bounces.enableSendgrid')">
+                        <b-switch v-model="form['bounce.sendgrid_enabled']"
+                          name="sendgrid_enabled" :native-value="true"
+                          data-cy="btn-enable-bounce-sendgrid" />
+                      </b-field>
+                    </div>
+                    <div class="column">
+                      <b-field :label="$t('settings.bounces.sendgridKey')"
+                        :message="$t('globals.messages.passwordChange')">
+                        <b-input v-model="form['bounce.sendgrid_key']" type="password"
+                          :disabled="!form['bounce.sendgrid_enabled']"
+                          name="sendgrid_enabled" :native-value="true"
+                          data-cy="btn-enable-bounce-sendgrid" />
+                      </b-field>
+                    </div>
+                  </div>
+              </div>
+            </div>
+
+            <!-- bounce mailbox -->
+            <b-field :label="$t('settings.bounces.enableMailbox')">
+              <b-switch v-model="form['bounce.mailboxes'][0].enabled"
+                :disabled="!form['bounce.enabled']"
+                name="enabled" :native-value="true" data-cy="btn-enable-bounce-mailbox" />
+            </b-field>
+
+            <template v-if="form['bounce.enabled'] && form['bounce.mailboxes'][0].enabled">
+              <div class="block box" v-for="(item, n) in form['bounce.mailboxes']" :key="n">
+                <div class="columns">
+                  <div class="column" :class="{'disabled': !item.enabled}">
+                    <div class="columns">
+                      <div class="column is-3">
+                        <b-field :label="$t('settings.bounces.type')" label-position="on-border">
+                          <b-select v-model="item.type" name="type">
+                              <option value="pop">POP</option>
+                          </b-select>
+                        </b-field>
+                      </div>
+                      <div class="column is-6">
+                        <b-field :label="$t('settings.bounces.host')" label-position="on-border"
+                          :message="$t('settings.bounces.hostHelp')">
+                          <b-input v-model="item.host" name="host"
+                            placeholder='bounce.yourmailserver.net' :maxlength="200" />
+                        </b-field>
+                      </div>
+                      <div class="column is-3">
+                        <b-field :label="$t('settings.bounces.port')" label-position="on-border"
+                          :message="$t('settings.bounces.portHelp')">
+                          <b-numberinput v-model="item.port" name="port" type="is-light"
+                              controls-position="compact"
+                              placeholder="25" min="1" max="65535" />
+                        </b-field>
+                      </div>
+                    </div><!-- host -->
+
+                    <div class="columns">
+                      <div class="column is-3">
+                        <b-field :label="$t('settings.bounces.authProtocol')"
+                          label-position="on-border">
+                          <b-select v-model="item.auth_protocol" name="auth_protocol">
+                            <option value="none">none</option>
+                            <option v-if="item.type === 'pop'" value="userpass">userpass</option>
+                            <template v-else>
+                              <option value="cram">cram</option>
+                              <option value="plain">plain</option>
+                              <option value="login">login</option>
+                            </template>
+                          </b-select>
+                        </b-field>
+                      </div>
+                      <div class="column">
+                        <b-field grouped>
+                          <b-field :label="$t('settings.bounces.username')"
+                            label-position="on-border" expanded>
+                            <b-input v-model="item.username"
+                              :disabled="item.auth_protocol === 'none'"
+                              name="username" placeholder="mysmtp" :maxlength="200" />
+                          </b-field>
+                          <b-field :label="$t('settings.bounces.password')"
+                            label-position="on-border" expanded
+                            :message="$t('settings.bounces.passwordHelp')">
+                            <b-input v-model="item.password"
+                              :disabled="item.auth_protocol === 'none'"
+                              name="password" type="password"
+                              :placeholder="$t('settings.bounces.passwordHelp')"
+                              :maxlength="200" />
+                          </b-field>
+                        </b-field>
+                      </div>
+                    </div><!-- auth -->
+
+                    <div class="columns">
+                      <div class="column is-6">
+                        <b-field grouped>
+                          <b-field :label="$t('settings.bounces.tls')" expanded
+                            :message="$t('settings.bounces.tlsHelp')">
+                            <b-switch v-model="item.tls_enabled" name="item.tls_enabled" />
+                          </b-field>
+                          <b-field :label="$t('settings.bounces.skipTLS')" expanded
+                            :message="$t('settings.bounces.skipTLSHelp')">
+                            <b-switch v-model="item.tls_skip_verify"
+                              :disabled="!item.tls_enabled" name="item.tls_skip_verify" />
+                          </b-field>
+                        </b-field>
+                      </div>
+                      <div class="column"></div>
+                      <div class="column is-4">
+                        <b-field :label="$t('settings.bounces.scanInterval')" expanded
+                          label-position="on-border"
+                          :message="$t('settings.bounces.scanIntervalHelp')">
+                          <b-input v-model="item.scan_interval" name="scan_interval"
+                            placeholder="15m" :pattern="regDuration" :maxlength="10" />
+                        </b-field>
+                      </div>
+                    </div><!-- TLS -->
+                  </div>
+                </div><!-- second container column -->
+              </div><!-- block -->
+            </template>
+          </b-tab-item><!-- bounces -->
+
           <b-tab-item :label="$t('settings.messengers.name')">
             <div class="items messengers">
               <div class="block box" v-for="(item, n) in form.messengers" :key="n">
@@ -583,6 +750,10 @@ export default Vue.extend({
       this.form.smtp.splice(i, 1);
     },
 
+    removeBounceBox(i) {
+      this.form['bounce.mailboxes'].splice(i, 1);
+    },
+
     showSMTPHeaders(i) {
       const s = this.form.smtp[i];
       s.showHeaders = true;
@@ -615,7 +786,7 @@ export default Vue.extend({
     onSubmit() {
       const form = JSON.parse(JSON.stringify(this.form));
 
-      // De-serialize custom e-mail headers.
+      // SMTP boxes.
       for (let i = 0; i < form.smtp.length; i += 1) {
         // If it's the dummy UI password placeholder, ignore it.
         if (form.smtp[i].password === dummyPassword) {
@@ -629,10 +800,22 @@ export default Vue.extend({
         }
       }
 
+      // Bounces boxes.
+      for (let i = 0; i < form['bounce.mailboxes'].length; i += 1) {
+        // If it's the dummy UI password placeholder, ignore it.
+        if (form['bounce.mailboxes'][i].password === dummyPassword) {
+          form['bounce.mailboxes'][i].password = '';
+        }
+      }
+
       if (form['upload.s3.aws_secret_access_key'] === dummyPassword) {
         form['upload.s3.aws_secret_access_key'] = '';
       }
 
+      if (form['bounce.sendgrid_key'] === dummyPassword) {
+        form['bounce.sendgrid_key'] = '';
+      }
+
       for (let i = 0; i < form.messengers.length; i += 1) {
         // If it's the dummy UI password placeholder, ignore it.
         if (form.messengers[i].password === dummyPassword) {
@@ -680,6 +863,12 @@ export default Vue.extend({
           d.smtp[i].password = dummyPassword;
         }
 
+        for (let i = 0; i < d['bounce.mailboxes'].length; i += 1) {
+          // The backend doesn't send passwords, so add a dummy so that
+          // the password looks filled on the UI.
+          d['bounce.mailboxes'][i].password = dummyPassword;
+        }
+
         for (let i = 0; i < d.messengers.length; i += 1) {
           // The backend doesn't send passwords, so add a dummy so that it
           // the password looks filled on the UI.
@@ -689,6 +878,7 @@ export default Vue.extend({
         if (d['upload.provider'] === 's3') {
           d['upload.s3.aws_secret_access_key'] = dummyPassword;
         }
+        d['bounce.sendgrid_key'] = dummyPassword;
 
         this.form = d;
         this.formCopy = JSON.stringify(d);

frontend/src/views/SubscriberForm.vue

@@ -2,7 +2,6 @@
   <form @submit.prevent="onSubmit">
     <div class="modal-card content" style="width: auto">
       <header class="modal-card-head">
-
         <b-tag v-if="isEditing" :class="[data.status, 'is-pulled-right']">{{ data.status }}</b-tag>
         <h4 v-if="isEditing">{{ data.name }}</h4>
         <h4 v-else>{{ $t('subscribers.newSubscriber') }}</h4>
@@ -12,25 +11,31 @@
           {{ $t('globals.fields.uuid') }}: {{ data.uuid }}
         </p>
       </header>
+
       <section expanded class="modal-card-body">
         <b-field :label="$t('subscribers.email')" label-position="on-border">
           <b-input :maxlength="200" v-model="form.email" name="email" :ref="'focus'"
             :placeholder="$t('subscribers.email')" required></b-input>
         </b-field>
 
-        <b-field :label="$t('globals.fields.name')" label-position="on-border">
+        <div class="columns">
-          <b-input :maxlength="200" v-model="form.name" name="name"
+          <div class="column is-8">
-            :placeholder="$t('globals.fields.name')"></b-input>
+            <b-field :label="$t('globals.fields.name')" label-position="on-border">
-        </b-field>
+              <b-input :maxlength="200" v-model="form.name" name="name"
-
+                :placeholder="$t('globals.fields.name')"></b-input>
-        <b-field :label="$t('globals.fields.status')" label-position="on-border"
+            </b-field>
-          :message="$t('subscribers.blocklistedHelp')">
+          </div>
-          <b-select v-model="form.status" name="status" :placeholder="$t('globals.fields.status')"
+          <div class="column is-4">
-            required>
+            <b-field :label="$t('globals.fields.status')" label-position="on-border"
-            <option value="enabled">{{ $t('subscribers.status.enabled') }}</option>
+              :message="$t('subscribers.blocklistedHelp')">
-            <option value="blocklisted">{{ $t('subscribers.status.blocklisted') }}</option>
+              <b-select v-model="form.status" name="status"
-          </b-select>
+                :placeholder="$t('globals.fields.status')" required expanded>
-        </b-field>
+                <option value="enabled">{{ $t('subscribers.status.enabled') }}</option>
+                <option value="blocklisted">{{ $t('subscribers.status.blocklisted') }}</option>
+              </b-select>
+            </b-field>
+          </div>
+        </div>
 
         <list-selector
           :label="$t('subscribers.lists')"
@@ -43,12 +48,48 @@
 
         <b-field :label="$t('subscribers.attribs')" label-position="on-border"
           :message="$t('subscribers.attribsHelp') + ' ' + egAttribs">
-          <b-input v-model="form.strAttribs" name="attribs" type="textarea" />
+          <div>
+            <b-input v-model="form.strAttribs" name="attribs" type="textarea" />
+            <a href="https://listmonk.app/docs/concepts"
+              target="_blank" rel="noopener noreferrer" class="is-size-7">
+              {{ $t('globals.buttons.learnMore') }} <b-icon icon="link-variant" size="is-small" />
+            </a>
+          </div>
         </b-field>
-        <a href="https://listmonk.app/docs/concepts"
+
-          target="_blank" rel="noopener noreferrer" class="is-size-7">
+        <div class="bounces" v-show="bounces.length > 0">
-          {{ $t('globals.buttons.learnMore') }} <b-icon icon="link" size="is-small" />.
+          <a href="#" class="is-size-6" disabed="true"
-        </a>
+            @click.prevent="toggleBounces">
+            <b-icon icon="email-bounce"></b-icon>
+            {{ $t('bounces.view') }} ({{ bounces.length }})
+          </a>
+          <a href="#" class="is-size-6 is-pulled-right" disabed="true"
+            @click.prevent="deleteBounces" v-if="isBounceVisible">
+            <b-icon icon="trash-can-outline"></b-icon>
+            {{ $t('globals.buttons.delete') }}
+          </a>
+
+          <div v-if="isBounceVisible" class="mt-4">
+            <ol class="is-size-7">
+              <li v-for="b in bounces" :key="b.id" class="mb-2">
+                  <div v-if="b.campaign">
+                    <router-link :to="{ name: 'bounces', query: { campaign_id: b.campaign.id } }">
+                      {{ b.campaign.name }}
+                    </router-link>
+                  </div>
+                  {{ $utils.niceDate(b.createdAt, true) }}
+                  <span class="is-pulled-right">
+                    <a href="#" @click.prevent="toggleMeta(b.id)">
+                      {{ b.source }}
+                      <b-icon :icon="visibleMeta[b.id] ? 'arrow-up' : 'arrow-down'" />
+                    </a>
+                  </span>
+                  <span class="is-clearfix"></span>
+                  <pre v-if="visibleMeta[b.id]">{{ b.meta }}</pre>
+              </li>
+            </ol>
+          </div>
+        </div>
       </section>
       <footer class="modal-card-foot has-text-right">
         <b-button @click="$parent.close()">{{ $t('globals.buttons.close') }}</b-button>
@@ -82,12 +123,45 @@ export default Vue.extend({
       // Binds form input values. This is populated by subscriber props passed
       // from the parent component in mounted().
       form: { lists: [], strAttribs: '{}' },
+      isBounceVisible: false,
+      bounces: [],
+      visibleMeta: {},
 
       egAttribs: '{"job": "developer", "location": "Mars", "has_rocket": true}',
     };
   },
 
   methods: {
+    toggleBounces() {
+      this.isBounceVisible = !this.isBounceVisible;
+    },
+
+    toggleMeta(id) {
+      let v = false;
+      if (!this.visibleMeta[id]) {
+        v = true;
+      }
+      Vue.set(this.visibleMeta, id, v);
+    },
+
+    deleteBounces(sub) {
+      this.$utils.confirm(
+        null,
+        () => {
+          this.$api.deleteSubscriberBounces(this.form.id).then(() => {
+            this.getBounces();
+            this.$utils.toast(this.$t('globals.messages.deleted', { name: sub.name }));
+          });
+        },
+      );
+    },
+
+    getBounces() {
+      this.$api.getSubscriberBounces(this.form.id).then((data) => {
+        this.bounces = data;
+      });
+    },
+
     onSubmit() {
       if (this.isEditing) {
         this.updateSubscriber();
@@ -183,6 +257,11 @@ export default Vue.extend({
       };
     }
 
+    if (this.form.id) {
+      this.getBounces();
+    }
+
+
     this.$nextTick(() => {
       this.$refs.focus.focus();
     });

frontend/src/views/Subscribers.vue

@@ -198,7 +198,8 @@
     </b-modal>
 
     <!-- Add / edit form modal -->
-    <b-modal scroll="keep" :aria-modal="true" :active.sync="isFormVisible" :width="600">
+    <b-modal scroll="keep" :aria-modal="true" :active.sync="isFormVisible" :width="600"
+      @close="onFormClose">
       <subscriber-form :data="curItem" :isEditing="isEditing"
         @finished="querySubscribers"></subscriber-form>
     </b-modal>
@@ -309,6 +310,12 @@ export default Vue.extend({
       this.isBulkListFormVisible = true;
     },
 
+    onFormClose() {
+      if (this.$route.params.id) {
+        this.$router.push({ name: 'subscribers' });
+      }
+    },
+
     onPageChange(p) {
       this.queryParams.page = p;
       this.querySubscribers();
@@ -472,8 +479,14 @@ export default Vue.extend({
       this.queryParams.listID = parseInt(this.$route.params.listID, 10);
     }
 
-    // Get subscribers on load.
+    if (this.$route.params.id) {
-    this.querySubscribers();
+      this.$api.getSubscriber(parseInt(this.$route.params.id, 10)).then((data) => {
+        this.showEditForm(data);
+      });
+    } else {
+      // Get subscribers on load.
+      this.querySubscribers();
+    }
   },
 });
 </script>

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/disintegration/imaging v1.6.2
 	github.com/gofrs/uuid v3.2.0+incompatible
 	github.com/jmoiron/sqlx v1.2.0
+	github.com/knadh/go-pop3 v0.1.0
 	github.com/knadh/goyesql/v2 v2.1.1
 	github.com/knadh/koanf v0.12.0
 	github.com/knadh/smtppool v0.2.1

go.sum

@@ -13,6 +13,10 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumC
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c=
 github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
+github.com/emersion/go-message v0.15.0 h1:urgKGqt2JAc9NFJcgncQcohHdiYb803YTH9OQwHBHIY=
+github.com/emersion/go-message v0.15.0/go.mod h1:wQUEfE+38+7EW8p8aZ96ptg6bAb1iwdgej19uXASlE4=
+github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 h1:IbFBtwoTQyw0fIM5xv1HF+Y+3ZijDR839WMulgxCcUY=
+github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594/go.mod h1:aqO8z8wPrjkscevZJFVE1wXJrLpC5LtJG7fqLOsPb2U=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
@@ -33,6 +37,8 @@ github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA=
 github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/knadh/go-pop3 v0.1.0 h1:MECWomq2uEGeuR7O2TjfzD63H47UFLKOqH1bSH7yhRU=
+github.com/knadh/go-pop3 v0.1.0/go.mod h1:a5kUJzrBB6kec+tNJl+3Z64ROgByKBdcyub+mhZMAfI=
 github.com/knadh/goyesql v2.0.0+incompatible h1:hJFJrU8kaiLmvYt9I/1k1AB7q+qRhHs/afzTfQ3eGqk=
 github.com/knadh/goyesql v2.0.0+incompatible/go.mod h1:W0tSzU8l7lYH1Fihj+bdQzkzOwvirrsMNHwkuY22qoY=
 github.com/knadh/goyesql/v2 v2.1.1 h1:Orp5ldaxPM4ozKHfu1m7p6iolJFXDGOpF3/jyOgO6ls=
@@ -117,8 +123,9 @@ golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200331124033-c3d80250170d h1:nc5K6ox/4lTFbMVSL9WRR81ixkcwXThoiF6yf+R9scA=
 golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

i18n/en.json

@@ -105,6 +105,7 @@
     "globals.buttons.close": "Close",
     "globals.buttons.continue": "Continue",
     "globals.buttons.delete": "Delete",
+    "globals.buttons.deleteAll": "Delete all",
     "globals.buttons.edit": "Edit",
     "globals.buttons.enabled": "Enabled",
     "globals.buttons.learnMore": "Learn more",
@@ -130,14 +131,17 @@
     "globals.messages.confirm": "Are you sure?",
     "globals.messages.created": "\"{name}\" created",
     "globals.messages.deleted": "\"{name}\" deleted",
+    "globals.messages.deletedCount": "{name} ({num}) deleted",
     "globals.messages.emptyState": "Nothing here",
+    "globals.messages.invalidData": "Invalid data",
+    "globals.messages.internalError": "Internal server error",
     "globals.messages.errorCreating": "Error creating {name}: {error}",
     "globals.messages.errorDeleting": "Error deleting {name}: {error}",
     "globals.messages.errorFetching": "Error fetching {name}: {error}",
     "globals.messages.errorUUID": "Error generating UUID: {error}",
     "globals.messages.errorUpdating": "Error updating {name}: {error}",
-    "globals.messages.invalidID": "Invalid ID",
+    "globals.messages.invalidID": "Invalid ID(s)",
-    "globals.messages.invalidUUID": "Invalid UUID",
+    "globals.messages.invalidUUID": "Invalid UUID(s)",
     "globals.messages.notFound": "{name} not found",
     "globals.messages.passwordChange": "Enter a value to change",
     "globals.messages.updated": "\"{name}\" updated",
@@ -164,6 +168,8 @@
     "globals.terms.settings": "Settings",
     "globals.terms.subscriber": "Subscriber | Subscribers",
     "globals.terms.subscribers": "Subscribers",
+    "globals.terms.bounce": "Bounce | Bounces",
+    "globals.terms.bounces": "Bounces",
     "globals.terms.tag": "Tag | Tags",
     "globals.terms.tags": "Tags",
     "globals.terms.template": "Template | Templates",
@@ -274,6 +280,21 @@
     "public.unsubbedInfo": "You have unsubscribed successfully.",
     "public.unsubbedTitle": "Unsubscribed",
     "public.unsubscribeTitle": "Unsubscribe from mailing list",
+    "bounces.unknownService": "Unknown service.",
+    "bounces.view": "View bounces",
+    "bounces.source": "Source",
+    "settings.bounces.name": "Bounces",
+    "settings.bounces.enable": "Enable bounce processing",
+    "settings.bounces.enableMailbox": "Enable bounce mailbox",
+    "settings.bounces.enableWebhooks": "Enable bounce webhooks",
+    "settings.bounces.count": "Bounce count",
+    "settings.bounces.countHelp": "Number of bounces per subscriber",
+    "settings.bounces.action": "Action",
+    "settings.bounces.blocklist": "Blocklist",
+    "settings.bounces.folder": "Folder",
+    "settings.bounces.folderHelp": "Name of the IMAP folder to scan. Eg: Inbox.",
+    "settings.bounces.delete": "Delete",
+    "settings.bounces.invalidScanInterval": "Bounce scan interval should be minimum 1 minute.",
     "settings.confirmRestart": "Ensure running campaigns are paused. Restart?",
     "settings.duplicateMessengerName": "Duplicate messenger name: {name}",
     "settings.errorEncoding": "Error encoding settings: {error}",
@@ -366,7 +387,7 @@
     "settings.smtp.idleTimeout": "Idle timeout",
     "settings.smtp.idleTimeoutHelp": "Time to wait for new activity on a connection before closing it and removing it from the pool (s for second, m for minute).",
     "settings.smtp.maxConns": "Max. connections",
-    "settings.smtp.maxConnsHelp": "Maximum concurrent connections to the SMTP server.",
+    "settings.smtp.maxConnsHelp": "Maximum concurrent connections to the server.",
     "settings.smtp.name": "SMTP",
     "settings.smtp.password": "Password",
     "settings.smtp.passwordHelp": "Enter to change",
@@ -382,6 +403,25 @@
     "settings.smtp.username": "Username",
     "settings.smtp.waitTimeout": "Wait timeout",
     "settings.smtp.waitTimeoutHelp": "Time to wait for new activity on a connection before closing it and removing it from the pool (s for second, m for minute).",
+    "settings.bounces.authProtocol": "Auth protocol",
+    "settings.bounces.type": "Type",
+    "settings.bounces.enabled": "Enabled",
+    "settings.bounces.host": "Host",
+    "settings.bounces.hostHelp": "Mail server's host address.",
+    "settings.bounces.scanInterval": "Scan interval",
+    "settings.bounces.scanIntervalHelp": "Interval at which the bounce mailbox should be scanned for bounces (s for second, m for minute).",
+    "settings.bounces.password": "Password",
+    "settings.bounces.passwordHelp": "Enter to change",
+    "settings.bounces.port": "Port",
+    "settings.bounces.portHelp": "Mail server's port.",
+    "settings.bounces.skipTLS": "Skip TLS verification",
+    "settings.bounces.skipTLSHelp": "Skip hostname check on the TLS certificate.",
+    "settings.bounces.tls": "TLS",
+    "settings.bounces.tlsHelp": "Enable STARTTLS.",
+    "settings.bounces.username": "Username",
+    "settings.bounces.enableSES": "Enable SES",
+    "settings.bounces.enableSendgrid": "Enable SendGrid",
+    "settings.bounces.sendgridKey": "SendGrid Key",
     "settings.title": "Settings",
     "settings.updateAvailable": "A new update {version} is available.",
     "subscribers.advancedQuery": "Advanced",

internal/bounce/bounce.go

@@ -0,0 +1,148 @@
+package bounce
+
+import (
+	"errors"
+	"log"
+	"time"
+
+	"github.com/jmoiron/sqlx"
+	"github.com/knadh/listmonk/internal/bounce/mailbox"
+	"github.com/knadh/listmonk/internal/bounce/webhooks"
+	"github.com/knadh/listmonk/models"
+	"github.com/lib/pq"
+)
+
+const (
+	// subID is the identifying subscriber ID header to look for in
+	// bounced e-mails.
+	subID  = "X-Listmonk-Subscriber"
+	campID = "X-Listmonk-Campaign"
+)
+
+// Mailbox represents a POP/IMAP mailbox client that can scan messages and pass
+// them to a given channel.
+type Mailbox interface {
+	Scan(limit int, ch chan models.Bounce) error
+}
+
+// Opt represents bounce processing options.
+type Opt struct {
+	BounceCount  int    `json:"count"`
+	BounceAction string `json:"action"`
+
+	MailboxEnabled  bool        `json:"mailbox_enabled"`
+	MailboxType     string      `json:"mailbox_type"`
+	Mailbox         mailbox.Opt `json:"mailbox"`
+	WebhooksEnabled bool        `json:"webhooks_enabled"`
+	SESEnabled      bool        `json:"ses_enabled"`
+	SendgridEnabled bool        `json:"sendgrid_enabled"`
+	SendgridKey     string      `json:"sendgrid_key"`
+}
+
+// Manager handles e-mail bounces.
+type Manager struct {
+	queue    chan models.Bounce
+	mailbox  Mailbox
+	SES      *webhooks.SES
+	Sendgrid *webhooks.Sendgrid
+	queries  *Queries
+	opt      Opt
+	log      *log.Logger
+}
+
+// Queries contains the queries.
+type Queries struct {
+	DB          *sqlx.DB
+	RecordQuery *sqlx.Stmt
+}
+
+// New returns a new instance of the bounce manager.
+func New(opt Opt, q *Queries, lo *log.Logger) (*Manager, error) {
+	m := &Manager{
+		opt:     opt,
+		queries: q,
+		queue:   make(chan models.Bounce, 1000),
+		log:     lo,
+	}
+
+	// Is there a mailbox?
+	if opt.MailboxEnabled {
+		switch opt.MailboxType {
+		case "pop":
+			m.mailbox = mailbox.NewPOP(opt.Mailbox)
+		case "imap":
+		default:
+			return nil, errors.New("unknown bounce mailbox type")
+		}
+	}
+
+	if opt.WebhooksEnabled {
+		if opt.SESEnabled {
+			m.SES = webhooks.NewSES()
+		}
+		if opt.SendgridEnabled {
+			sg, err := webhooks.NewSendgrid(opt.SendgridKey)
+			if err != nil {
+				lo.Printf("error initializing sendgrid webhooks: %v", err)
+			} else {
+				m.Sendgrid = sg
+			}
+		}
+	}
+
+	return m, nil
+}
+
+// Run is a blocking function that listens for bounce events from webhooks and or mailboxes
+// and executes them on the DB.
+func (m *Manager) Run() {
+	if m.opt.MailboxEnabled {
+		go m.runMailboxScanner()
+	}
+
+	for {
+		select {
+		case b, ok := <-m.queue:
+			if !ok {
+				return
+			}
+
+			_, err := m.queries.RecordQuery.Exec(b.SubscriberUUID,
+				b.Email,
+				b.CampaignUUID,
+				b.Type,
+				b.Source,
+				b.Meta,
+				b.CreatedAt,
+				m.opt.BounceCount,
+				m.opt.BounceAction)
+			if err != nil {
+				// Ignore the error if it complained of no subscriber.
+				if pqErr, ok := err.(*pq.Error); ok && pqErr.Column == "subscriber_id" {
+					m.log.Printf("bounced subscriber (%s / %s) not found", b.SubscriberUUID, b.Email)
+					continue
+				}
+				m.log.Printf("error recording bounce: %v", err)
+			}
+		}
+	}
+}
+
+// runMailboxScanner runs a blocking loop that scans the mailbox at given intervals.
+func (m *Manager) runMailboxScanner() {
+	for {
+		if err := m.mailbox.Scan(1000, m.queue); err != nil {
+			m.log.Printf("error scanning bounce mailbox: %v", err)
+		}
+
+		time.Sleep(m.opt.Mailbox.ScanInterval)
+	}
+}
+
+// Record records a new bounce event given the subscriber's email or UUID.
+func (m *Manager) Record(b models.Bounce) error {
+	select {
+	case m.queue <- b:
+	}
+	return nil
+}

internal/bounce/gmail.bounce

@@ -0,0 +1,257 @@
+Delivered-To: kailash@zerodha.com
+Received: by 2002:a54:21c4:0:0:0:0:0 with SMTP id i4csp2867282eco;
+        Sun, 23 May 2021 10:33:16 -0700 (PDT)
+X-Received: by 2002:a17:902:bb87:b029:ef:1ef:b4a5 with SMTP id m7-20020a170902bb87b02900ef01efb4a5mr21801783pls.28.1621791195832;
+        Sun, 23 May 2021 10:33:15 -0700 (PDT)
+ARC-Seal: i=1; a=rsa-sha256; t=1621791195; cv=none;
+        d=google.com; s=arc-20160816;
+        b=xZixRTOHnpK7AKFJqDRGvXg8csiC/HDweapqiROpH9f3CBKOp2bNxesRYQAhF9dRER
+         TcIdmsNBWmAsM3UCrKP1gsafEEhLa/egWet5tS7eRVNtrlf4xIr/Oyizzi/+vWTaYBaj
+         SYS6ig0kEx1TIu23fhipMkjmqpba1CvekFt0Sujn51Wl/pCbxQLwXUUG+F2NlOZFnMNy
+         GkxHgi+2lRqeowzPxMaUxat6yD0uym7V0TephJhTPTekZzIrXHQTd1T023qyvfUjdLU9
+         HtXjkqJpJ2NIsHwhLTDqC860/dJMpKhMt6ekH5wK7ooXyXylOIeE9z9grYVVX7esWGgv
+         M7Tw==
+ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816;
+        h=in-reply-to:references:subject:from:date:message-id:auto-submitted
+         :to:dkim-signature;
+        bh=N5pKD3v8h0to825sNHx1obZB8CFEVlHQ1rrjBil1NtM=;
+        b=XNDLUAQC40qw4KX91RFgUOKWbgdsyQz86pvi7wENg/xBasGRJJjyZgBTYyA2e8XUN4
+         3pbUZG68HsGI1bAE/t5HefjTmHRtoSh/nZzMVk+hoHbeFPtMcOo9sDWhcWnZjfFE2tzU
+         lEDFV1M1NeKf8JcW+nm7Sq6haAv/M7C9q++kQxt0P6GnU17IOb5DyeUQ9SRVa1mTgjZt
+         TuL94m2a7N6/KkHRrQCVyd1SZJR4+JDhFbdoScc0GXmu+aCt0DlznymAiLRX6SLB/sbx
+         B4Aj9luupUg/9yzy0JCZ9qhjY3w+36mcz9EnIuA6TJP1AmBUFiVHVXjLTz8FKhKP/TcN
+         Szrw==
+ARC-Authentication-Results: i=1; mx.google.com;
+       dkim=pass header.i=@googlemail.com header.s=20161025 header.b="d/j1xe99";
+       spf=pass (google.com: best guess record for domain of postmaster@mail-sor-f69.google.com designates 209.85.220.69 as permitted sender) smtp.helo=mail-sor-f69.google.com;
+       dmarc=pass (p=QUARANTINE sp=QUARANTINE dis=NONE) header.from=googlemail.com
+Return-Path: <>
+Received: from mail-sor-f69.google.com (mail-sor-f69.google.com. [209.85.220.69])
+        by mx.google.com with SMTPS id x9sor5739157pjh.37.2021.05.23.10.33.15
+        for <kailash@zerodha.com>
+        (Google Transport Security);
+        Sun, 23 May 2021 10:33:15 -0700 (PDT)
+Received-SPF: pass (google.com: best guess record for domain of postmaster@mail-sor-f69.google.com designates 209.85.220.69 as permitted sender) client-ip=209.85.220.69;
+Authentication-Results: mx.google.com;
+       dkim=pass header.i=@googlemail.com header.s=20161025 header.b="d/j1xe99";
+       spf=pass (google.com: best guess record for domain of postmaster@mail-sor-f69.google.com designates 209.85.220.69 as permitted sender) smtp.helo=mail-sor-f69.google.com;
+       dmarc=pass (p=QUARANTINE sp=QUARANTINE dis=NONE) header.from=googlemail.com
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
+        d=googlemail.com; s=20161025;
+        h=to:auto-submitted:message-id:date:from:subject:references
+         :in-reply-to;
+        bh=N5pKD3v8h0to825sNHx1obZB8CFEVlHQ1rrjBil1NtM=;
+        b=d/j1xe99m2/tEFKHFbaxWYQDypZPPXM4yaLLXfv78vQTkusHx6ezzjbbLDQThRi4Gp
+         icphSZc7bxQqNICr6VHhbaFiywhQUwKRd4Atrv6pFO2pRJAPUX0U9NOK1ktRNsd5ePUA
+         sIHIwA346yYp2mVsjzKaoAO6hmKG9wota8RYPKE2n3zHQPKdv+TzM9C/r1ddBrcyd92f
+         tQeM0ySxXyPoQBvHmBhQyM02QcdB43GI7MqChsMM55FsyOAkzSyk2mpr/fR4WRFzqizB
+         soso7v4Fk4SGSL/YWirMBEYfV4lMC02as8s+C/T3cDCDSLSYevk9TzQZkJCesxGg9v7Y
+         oZsw==
+X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
+        d=1e100.net; s=20161025;
+        h=x-gm-message-state:to:auto-submitted:message-id:date:from:subject
+         :references:in-reply-to;
+        bh=N5pKD3v8h0to825sNHx1obZB8CFEVlHQ1rrjBil1NtM=;
+        b=OrbtNPg5Os277o6RgpirMIAKmj2lOd2QCNP4GoWd587x5FG6IkDGS0FMlgqir4kd1m
+         onrouDFZ2YIfwYjes6UwVc/jRtFEqweNfW148Wwtr0da3N++Q722KoKaIlivsi6tRic5
+         IpLG2R/AdMRHyqKgc+32sY5bFpxA8dvV8QzeSjMDOiEW6MDW+5EPZRvlmrQHcs3gltgw
+         fu24ZBvxO5p+SVA3fv4CHofWBx41eff1xDj3YeXBZC3UvycM3uDXPCccNSdGlFVdVj1k
+         ShVVBaMYe7ogQaE7NuBI2fTg+2Gwj8A1qaZWkqNa5WMsV+IBGn5X+9hGv5+a2oRBuML2
+         VYhw==
+X-Gm-Message-State: AOAM532KXW9W45VWVICwrMKyOmBmBlxCXVqvZLIyub147S7L81Y7Yop3
+	eSoPXNYLWGr8Yfu3nAAx9bqznTMyYnI5amD+uBfPOw==
+X-Google-Smtp-Source: ABdhPJyoLDp5/aV12uOu5ZlEUItHwuaalyHiLdxJo/DQjtVQnyFAt10Oz/cnjtoGwe2hdi+YlKG+Ouy2NlxgqA9gI9fMRnAyaqs9KtM=
+X-Received: by 2002:a17:90b:30d0:: with SMTP id hi16mr20431437pjb.30.1621791195594;
+        Sun, 23 May 2021 10:33:15 -0700 (PDT)
+Content-Type: multipart/report; boundary="0000000000001cf3b205c302b0a3"; report-type=delivery-status
+To: kailash@zerodha.com
+Received: by 2002:a17:90b:30d0:: with SMTP id hi16mr15665871pjb.30; Sun, 23
+ May 2021 10:33:15 -0700 (PDT)
+Return-Path: <>
+Auto-Submitted: auto-replied
+Message-ID: <60aa91db.1c69fb81.218cf.a9fa.GMR@mx.google.com>
+Date: Sun, 23 May 2021 10:33:15 -0700 (PDT)
+From: Mail Delivery Subsystem <mailer-daemon@googlemail.com>
+Subject: Delivery Status Notification (Failure)
+References: <b417b131-afeb-1989-0d49-d6114141a784@zerodha.com>
+In-Reply-To: <b417b131-afeb-1989-0d49-d6114141a784@zerodha.com>
+X-Failed-Recipients: psaodp2apsdoad9@zerodha.com
+
+--0000000000001cf3b205c302b0a3
+Content-Type: multipart/related; boundary="0000000000001cfdaa05c302b0a7"
+
+--0000000000001cfdaa05c302b0a7
+Content-Type: multipart/alternative; boundary="0000000000001cfdb405c302b0a8"
+
+--0000000000001cfdb405c302b0a8
+Content-Type: text/plain; charset="UTF-8"
+
+
+** Address not found **
+
+Your message wasn't delivered to psaodp2apsdoad9@zerodha.com because the address couldn't be found, or is unable to receive mail.
+
+Learn more here: https://support.google.com/mail/?p=NoSuchUser
+
+The response was:
+
+550 5.1.1 The email account that you tried to reach does not exist. Please try double-checking the recipient's email address for typos or unnecessary spaces. Learn more at https://support.google.com/mail/?p=NoSuchUser j11sor9289118pjs.32 - gsmtp
+
+--0000000000001cfdb405c302b0a8
+Content-Type: text/html; charset="UTF-8"
+
+
+<html>
+<head>
+<style>
+* {
+font-family:Roboto, "Helvetica Neue", Helvetica, Arial, sans-serif;
+}
+</style>
+</head>
+<body>
+<table cellpadding="0" cellspacing="0" class="email-wrapper" style="padding-top:32px;background-color:#ffffff;"><tbody>
+<tr><td>
+<table cellpadding=0 cellspacing=0><tbody>
+<tr><td style="max-width:560px;padding:24px 24px 32px;background-color:#fafafa;border:1px solid #e0e0e0;border-radius:2px">
+<img style="padding:0 24px 16px 0;float:left" width=72 height=72 alt="Error Icon" src="cid:icon.png">
+<table style="min-width:272px;padding-top:8px"><tbody>
+<tr><td><h2 style="font-size:20px;color:#212121;font-weight:bold;margin:0">
+Address not found
+</h2></td></tr>
+<tr><td style="padding-top:20px;color:#757575;font-size:16px;font-weight:normal;text-align:left">
+Your message wasn't delivered to <a style='color:#212121;text-decoration:none'><b>psaodp2apsdoad9@zerodha.com</b></a> because the address couldn't be found, or is unable to receive mail.
+</td></tr>
+<tr><td style="padding-top:24px;color:#4285F4;font-size:14px;font-weight:bold;text-align:left">
+<a style="text-decoration:none" href="https://support.google.com/mail/?p=NoSuchUser">LEARN MORE</a>
+</td></tr>
+</tbody></table>
+</td></tr>
+</tbody></table>
+</td></tr>
+<tr style="border:none;background-color:#fff;font-size:12.8px;width:90%">
+<td align="left" style="padding:48px 10px">
+The response was:<br/>
+<p style="font-family:monospace">
+550 5.1.1 The email account that you tried to reach does not exist. Please try double-checking the recipient&#39;s email address for typos or unnecessary spaces. Learn more at https://support.google.com/mail/?p=NoSuchUser j11sor9289118pjs.32 - gsmtp
+</p>
+</td>
+</tr>
+</tbody></table>
+</body>
+</html>
+
+--0000000000001cfdb405c302b0a8--
+--0000000000001cfdaa05c302b0a7
+Content-Type: image/png; name="icon.png"
+Content-Disposition: attachment; filename="icon.png"
+Content-Transfer-Encoding: base64
+Content-ID: <icon.png>
+
+iVBORw0KGgoAAAANSUhEUgAAAJAAAACQCAYAAADnRuK4AAAACXBIWXMAABYlAAAWJQFJUiTwAAAA
+GXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAABTdJREFUeNrsnD9sFEcUh5+PRMqZ
+yA0SPhAUQAQFUkyTgiBASARo6QApqVIkfdxGFJFSgGhJAUIiBaQB0ZIOKVCkwUgURjIg2fxL4kS+
+YDvkbC/388bi8N16Z4/d7J/5PsniuD3fyePP772ZeTsDQRAYQL/UGAJAIEAgQCBAIAAEAgQCBAIE
+AkAgyJT3Mv+Eq7vYK8mTE+MDRCAghQECAeRQA5V2ZOpmg5vDx3NPzRbmGRMEcmTrEbNNB8zWfRD+
+f/Efs2e3zCZvMjaksBg27TfbcuSNPEKP9ZyuAQKtHX2O9ncNgWC57umMPKvRNb0GEKgnLoUyxTQC
+rcns0/6uIRAs8/hGf9cQCJZpTpjdO2f25/03z+mxntM1eLtsZAgiUtX4JcaBCAQIBAgECARQ8CJa
+G5jab4J4pm4WZmO3OALVh802fIwcLkyPkcKAGggAgQCBAIEAgQCBABAIEAjKA/1AnahhbO5FdOOY
+VsrrDbPBYcYKgf5D2wLaV3p+22xh1u17tO3S+DTcvxvagUDeivPgx/a/95J/73w7Sj26Hn4pKo2M
+ehuV/KyBJM6d0f7k6RKx/R63vvL2tmf/ItDdM2ZTP6f7nkp9Y2fDx1v9akmpIU+KSCLVUghUQfSL
+zVKeTklbLxGoctw/nzC5rw8L5KRNbkpnKq6pgSqEClzNnFzY+XnYWrt6VpVk1vbwWvg+RKCKMOUw
+Q1LEOXA+/MX3mpJvGDHb265xtnzmFoUK1HaKQGlMtePYM+q2KKjXuaS1NJYIEKgI8jhEgqHt4cqy
+Ky53j3hyHz2bqSLp2o2LbJ7MxKovkGqXteoWpaOk96O9/yF/dF7NwlS36AuIQIBA5celQK4PIxBE
+4LLzrtoLgaALdSy6CJRkWQCBPGLsTHznomZ9nszUECgJ2ml3WWHe+QVFNPSQx6UdZNtxr9pbEShN
+eTTz8mQXHoHSlke7+Z+c9m6VGoHSkEfs/trLW3wQKApN1V3lGfnGu2Z6BFoLtYCs3GWBPAiUCLVh
+/HoaeRCoT9R873KLM/IgUBfapnCpe5AHgXry4pf412ihEHkQqCdxd5VqrcezhUIESsJMTJ+Pdthp
+Z0WgyNlXXPHc2Mc4IVAELl2Gnh8mhUDvCkfbIVAkcbf/aOoO3fMKhqAD3frTa4quwpn0hUDOkQhI
+YYBAgECAQAAU0QlYObl+5Ug8NcprZkZxjUCxRPVA6zmtEXHCBykskrhjgHXN09PoEcgFl4M4H11j
+nBAoApcj6ZoPGScEAgTKApcDoTw5sgWB+sGlz1n90IBAPdE6j1o21PfcC11jLagL1oFWRyGlKU3p
+OxcSJQ7NZAjkhHp/uG2HFAYIBAgECASAQIBAgECAQAAIBOkxEARBtp9wdVfAMOfIifEBIhCQwgCB
+ABAI0oV2jhxZ+nfBatuPZfgBCy0Eqqo8c01b+uu51XZvzOgDWoHNTGR+pCwpLEd5svuAZXlO2uEr
+PyEQ8hRWHgRCHmqg0sjTnLalv6crJQ8C/U8stqNO0I4+VZOHFIY8COS1PGL2ybd5yUMKK7s8zYmL
+dujyd3n+nESgcsvzZd4/KwIhDwIhT35QA6UyE1qyxZnfvJMHgdKS549JC1qvvJOHFIY8CFR5eV5O
+XimqPAhUdHnmfx+zgxdOFXkoqIGKKs/cswnb/8Oeog8HEai48nxUhiFBIORBIOShBioskkbySCLk
+IQIhDwIhj28p7FApR6b1qlEbHGpkO/rr6215vi/zH1r2x7tApSGFAQIBAgECAQIBIBAgECAQIBBA
+LK8FGADCTxYrr+EVJgAAAABJRU5ErkJggg==
+--0000000000001cfdaa05c302b0a7--
+--0000000000001cf3b205c302b0a3
+Content-Type: message/delivery-status
+
+Reporting-MTA: dns; googlemail.com
+Received-From-MTA: dns; kailash@zerodha.com
+Arrival-Date: Sun, 23 May 2021 10:33:14 -0700 (PDT)
+X-Original-Message-ID: <b417b131-afeb-1989-0d49-d6114141a784@zerodha.com>
+
+Final-Recipient: rfc822; psaodp2apsdoad9@zerodha.com
+Action: failed
+Status: 5.1.1
+Diagnostic-Code: smtp; 550-5.1.1 The email account that you tried to reach does not exist. Please try
+ 550-5.1.1 double-checking the recipient's email address for typos or
+ 550-5.1.1 unnecessary spaces. Learn more at
+ 550 5.1.1  https://support.google.com/mail/?p=NoSuchUser j11sor9289118pjs.32 - gsmtp
+Last-Attempt-Date: Sun, 23 May 2021 10:33:15 -0700 (PDT)
+
+--0000000000001cf3b205c302b0a3
+Content-Type: message/rfc822
+
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
+        d=zerodha.com; s=google;
+        h=to:from:subject:message-id:date:user-agent:mime-version
+         :content-language:content-transfer-encoding;
+        bh=lzgCIA1dMfq+1Zoi8Mtush+Jqj/mDmwC2uBHfa8ybDc=;
+        b=OI+HpFZBSgaEofYQU9PrR5WymG/k8EXLOh0LJTaCLBt+fyv9xRqmIPJQwHaPaoV3o5
+         a3YM9Lbq14BGK8ySHp+ffBcony8TiyqFEa61ostQvQyE21YayJg6EdacY/xHwpFlf8qP
+         H7iBkJp1pMztZEyxwgu3dIKLkSicVMMlQVEVHpMhq6qaaypTc1VDQab4o9DB0/QPGmXV
+         RJGbXn+UOLpY+sxxBrxYa65cszT9gbhIxXSB30SsRW3p7ZbtIEouaat7x4QIc0FCPfnc
+         aQG8o0qFMQmaGbTvaGN4GdMPB/wBjfbhDqxG+uRTETQ75hcE7Pd1ymcivHjuwb8MxAgR
+         3VhQ==
+X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
+        d=1e100.net; s=20161025;
+        h=x-gm-message-state:to:from:subject:message-id:date:user-agent
+         :mime-version:content-language:content-transfer-encoding;
+        bh=lzgCIA1dMfq+1Zoi8Mtush+Jqj/mDmwC2uBHfa8ybDc=;
+        b=uZOZm6bHzCDq0BNBWANQbLYgwTFfmAe4jbJMeMyD118JH2ygj6RZgIoXdf5RsxL2DF
+         Cj4FeXDRUPzl34SS9hshgD0fLJeLCxiKhRZZK7cuWrjelTY6A4zjNv5U3Z5+3EWk75p8
+         tmnHxk2w86TDitiS/NH2MVPhjou20iwAW7KNlWLlvi0W7DO/1eK5zonfuAMMR8uUCV0F
+         YGtz/WgHVnY//gFOhCFpGLxVBm+U8QGEigG8MLDUiGpc9lmvtwkMkpvnO0UZSeITYAlk
+         xCN0Jk30pgBEq6CJ1m5TmqeAft7fv6258M/0TH5L1EVqAJQ+9wpvTvPg4FQQrrwz7/PK
+         VOpg==
+X-Gm-Message-State: AOAM532L5lDl9xQIf19Fc0oe2hyHLx8Em+K7lIllieBybFHp01Zr2H3J
+	wh64f6L6IgQ5tPBvmlRH8IctB4IiHdZquJoRp10FWHKjn7+L4jib5wsUDVnM/Uyjw44b326R08D
+	fC4vnTb41a9b4AAoJSbKzgIB01Qy81YNUFx7qu9SSQBmaDxggTl/IQt6VsxdVeaY63SMd8rsnlm
+	FmEFg=
+X-Google-Smtp-Source: ABdhPJzV4UdWvj19F/ju1ONILTdIJGjh0CEXGZRIXb0obPTeAAQoIvIxw87mW78rKDpFnLzRgUZhMw==
+X-Received: by 2002:a17:90b:30d0:: with SMTP id hi16mr20431400pjb.30.1621791194884;
+        Sun, 23 May 2021 10:33:14 -0700 (PDT)
+Return-Path: <kailash@zerodha.com>
+Received: from [192.168.1.108] ([106.51.89.95])
+        by smtp.gmail.com with ESMTPSA id n23sm6419900pgv.76.2021.05.23.10.33.13
+        for <psaodp2apsdoad9@zerodha.com>
+        (version=TLS1_3 cipher=TLS_AES_128_GCM_SHA256 bits=128/128);
+        Sun, 23 May 2021 10:33:14 -0700 (PDT)
+To: psaodp2apsdoad9@zerodha.com
+From: Kailash Nadh <kailash@zerodha.com>
+Subject: test
+Message-ID: <b417b131-afeb-1989-0d49-d6114141a784@zerodha.com>
+Date: Sun, 23 May 2021 23:03:11 +0530
+User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:78.0) Gecko/20100101
+ Thunderbird/78.10.1
+MIME-Version: 1.0
+Content-Type: text/html; charset=utf-8
+Content-Language: en-US
+Content-Transfer-Encoding: 7bit
+
+<html>
+  <head>
+
+    <meta http-equiv="content-type" content="text/html; charset=UTF-8">
+  </head>
+  <body>
+    <p><font size="-1"><font face="Arial">this is a test message and can
+          be ignored.</font></font><br>
+    </p>
+  </body>
+</html>
+
+--0000000000001cf3b205c302b0a3--

internal/bounce/mailbox/opt.go

@@ -0,0 +1,29 @@
+package mailbox
+
+import "time"
+
+// Opt represents an e-mail POP/IMAP mailbox configuration.
+type Opt struct {
+	// Host is the server's hostname.
+	Host string `json:"host"`
+
+	// Port is the server port.
+	Port int `json:"port"`
+
+	AuthProtocol string `json:"auth_protocol"`
+
+	// Username is the mail server login username.
+	Username string `json:"username"`
+
+	// Password is the mail server login password.
+	Password string `json:"password"`
+
+	// Folder is the name of the IMAP folder to scan for e-mails.
+	Folder string `json:"folder"`
+
+	// Optional TLS settings.
+	TLSEnabled    bool `json:"tls_enabled"`
+	TLSSkipVerify bool `json:"tls_skip_verify"`
+
+	ScanInterval time.Duration `json:"scan_interval"`
+}

internal/bounce/mailbox/pop.go

@@ -0,0 +1,119 @@
+package mailbox
+
+import (
+	"encoding/json"
+	"time"
+
+	"github.com/knadh/go-pop3"
+	"github.com/knadh/listmonk/models"
+)
+
+// POP represents a POP mailbox.
+type POP struct {
+	opt    Opt
+	client *pop3.Client
+}
+
+// NewPOP returns a new instance of the POP mailbox client.
+func NewPOP(opt Opt) *POP {
+	return &POP{
+		opt: opt,
+		client: pop3.New(pop3.Opt{
+			Host:          opt.Host,
+			Port:          opt.Port,
+			TLSEnabled:    opt.TLSEnabled,
+			TLSSkipVerify: opt.TLSSkipVerify,
+		}),
+	}
+}
+
+// Scan scans the mailbox and pushes the downloaded messages into the given channel.
+// The messages that are downloaded are deleted from the server. If limit > 0,
+// all messages on the server are downloaded and deleted.
+func (p *POP) Scan(limit int, ch chan models.Bounce) error {
+	c, err := p.client.NewConn()
+	if err != nil {
+		return err
+	}
+	defer c.Quit()
+
+	// Authenticate.
+	if p.opt.AuthProtocol != "none" {
+		if err := c.Auth(p.opt.Username, p.opt.Password); err != nil {
+			return err
+		}
+	}
+
+	// Get the total number of messages on the server.
+	count, _, err := c.Stat()
+	if err != nil {
+		return err
+	}
+
+	// No messages.
+	if count == 0 {
+		return nil
+	}
+
+	if limit > 0 && count > limit {
+		count = limit
+	}
+
+	// Download messages.
+	for id := 1; id <= count; id++ {
+		// Download just one line of the body as the body is not required at all.
+		m, err := c.Top(id, 1)
+		if err != nil {
+			return err
+		}
+
+		var (
+			campUUID = m.Header.Get(models.EmailHeaderCampaignUUID)
+			subUUID  = m.Header.Get(models.EmailHeaderSubscriberUUID)
+			date, _  = time.Parse("Mon, 02 Jan 2006 15:04:05 -0700", m.Header.Get("Date"))
+		)
+
+		if campUUID == "" || subUUID == "" {
+			continue
+		}
+		if date.IsZero() {
+			date = time.Now()
+		}
+
+		// Additional bounce e-mail metadata.
+		meta, _ := json.Marshal(struct {
+			From        string   `json:"from"`
+			Subject     string   `json:"subject"`
+			MessageID   string   `json:"message_id"`
+			DeliveredTo string   `json:"delivered_to"`
+			Received    []string `json:"received"`
+		}{
+			From:        m.Header.Get("From"),
+			Subject:     m.Header.Get("Subject"),
+			MessageID:   m.Header.Get("Message-Id"),
+			DeliveredTo: m.Header.Get("Delivered-To"),
+			Received:    m.Header.Map()["Received"],
+		})
+
+		select {
+		case ch <- models.Bounce{
+			Type:           "hard",
+			CampaignUUID:   m.Header.Get(models.EmailHeaderCampaignUUID),
+			SubscriberUUID: m.Header.Get(models.EmailHeaderSubscriberUUID),
+			Source:         p.opt.Host,
+			CreatedAt:      date,
+			Meta:           json.RawMessage(meta),
+		}:
+		default:
+		}
+	}
+
+	// Delete the downloaded messages.
+	for id := 1; id <= count; id++ {
+		if err := c.Dele(id); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

internal/bounce/proton.bounce

@@ -0,0 +1,152 @@
+Content-Type: multipart/mixed;
+ boundary=ea9649a4b8e544897b542b3a1b97e2d172a7efdd385fe08531c312b947091452
+References:  <_GK6cYA-I3T01OCdH_MIi1whoJheEQ3--oTPqsatnNKqyJTNfcCFOQMJRqXyDtdOn_jPw7qb40E1F8QlDS_OAw==@protonmail.internalid>
+X-Pm-Date: Sun, 23 May 2021 17:12:42 +0000
+X-Pm-External-Id: <4Fp6NV4QXQz4wwcG@mail1.protonmail.ch>
+X-Pm-Internal-Id: _GK6cYA-I3T01OCdH_MIi1whoJheEQ3--oTPqsatnNKqyJTNfcCFOQMJRqXyDtdOn_jPw7qb40E1F8QlDS_OAw==
+To: <kailash@nadh.in>
+Reply-To: <MAILER-DAEMON@protonmail.com>
+From: <MAILER-DAEMON@protonmail.com>
+Subject: Undelivered Mail Returned to Sender
+Delivered-To: kailash@nadh.in
+X-Pm-Spam-Action: dunno
+X-Pm-Origin: internal
+X-Attached: email-1.3.eml
+Return-Path: <>
+Mime-Version: 1.0
+Authentication-Results: mailin027.protonmail.ch; dkim=none
+Authentication-Results: mailin027.protonmail.ch; spf=pass
+ smtp.helo=mail-40136.protonmail.ch
+Authentication-Results: mailin027.protonmail.ch; dmarc=fail (p=quarantine
+ dis=none) header.from=protonmail.com
+Received: by mail1.protonmail.ch (Postfix) id 4Fp6NV4QXQz4wwcG; Sun, 23 May
+ 2021 17:12:42 +0000 (UTC)
+Received: from mail-40136.protonmail.ch (mail-40136.protonmail.ch
+ [185.70.40.136]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256
+ bits) key-exchange X25519 server-signature RSA-PSS (4096 bits)
+ server-digest SHA256) (No client certificate requested) by
+ mailin027.protonmail.ch (Postfix) with ESMTPS id 4Fp6NV73rqz9vNPR for
+ <kailash@nadh.in>; Sun, 23 May 2021 17:12:42 +0000 (UTC)
+X-Pm-Spam: 0yeiAIic37iBOIJChpR3Y2bi4AiOiuHVZb8miiACL3cpJI6ZC2CIIMFGhwQGOmY2Y
+ E4MWxDYMNUDzzQWOzMiA0sIHzCJIYIS6gsHImIzlNwX3iW0YOAiwiACL2cvNUicmziAOLACiwVmc 3b0JogIjujIIMBCCFlVQ1U18BCMCZ0RTOBiCllXYyczBFtcGyHBIbJ2hslmYXa5RlzIGwDQIIRHv wYDIVJuxtzIFy29YZoTg04CMTMwMxuXVuTALMBCSEZ1Q0XOl1TX0LElUR9VISByMkQ6wdvIEg2Qb cVmwhRXdGdvlAobipzMKX4Gg4EzWSN3440MCxC4MMYzgzlGbGdkVluIGsHdIL1mhzxWaGcrl5uZS dXRZX4Ggw4CMEISZVNRUMUlQXZ0Sg00T2UuVVyZGtGVIYlWszlGIGIvN1vbW5mxbIFGilNXdCZlB R1bmy2VcI1GhcxWaibwB92cmlWRacxluttFIWYslItZXlGFZb9Wu0F2WXXyBRvb3hm1bawWut92Y VXuxAuLTTCBMUZEfMVES1TQ9NTQVQFNIRojgMVESyTtBRjYXzGVaINFQyBiRWZvNRccmwi0bLAjg WNkUFRJ99NTlJ1BUSV0fgw0VWTpFNwbHlWtaIdGvgQ2b2cuVVyZGu1xcIEDuTBSOEUP9VERkS0ZX RVUNMlUQEIv5RlIGy2NcaBX0u9WaGI2FlsYWsWJYZ5ScgIibSf9B
+Date: Sun, 23 May 2021 17:12:42 +0000 (UTC)
+X-Original-To: kailash@nadh.in
+X-Pm-Content-Encryption: on-delivery
+Message-Id: <4Fp6NV4QXQz4wwcG@mail1.protonmail.ch>
+Auto-Submitted: auto-replied
+X-Pm-Spamscore: 0
+
+--ea9649a4b8e544897b542b3a1b97e2d172a7efdd385fe08531c312b947091452
+Content-Transfer-Encoding: quoted-printable
+Content-Type: text/plain; charset=utf-8
+
+This is the mail system at ProtonMail.
+
+I'm sorry to have to inform you that your message could not
+be delivered to one or more recipients. It's attached below.
+
+For further assistance, please forward this email to support@protonmail.com=
+.
+
+If you do so, you can delete your own text from the attached returned messa=
+ge.
+
+
+<osdktestpaostestapleps199aopp@zerodha.com>: host
+    aspmx.l.google.com[172.217.218.26] said: 550-5.1.1 The email account th=
+at
+    you tried to reach does not exist. Please try 550-5.1.1 double-checking=
+ the
+    recipient's email address for typos or 550-5.1.1 unnecessary spaces. Le=
+arn
+    more at 550 5.1.1  https://support.google.com/mail/?p=3DNoSuchUser
+    c95si12273060edf.464 - gsmtp (in reply to RCPT TO command)
+----------------------------------------------
+message/delivery-status
+----------------------------------------------
+Reporting-MTA: dns; mail1.protonmail.ch
+X-Postfix-Queue-ID: 4Fp6NT5QDfz4wwcW
+X-Postfix-Sender: rfc822; kailash@nadh.in
+Arrival-Date: Sun, 23 May 2021 17:12:41 +0000 (UTC)
+
+Final-Recipient: rfc822; osdktestpaostestapleps199aopp@zerodha.com
+Original-Recipient: rfc822;osdktestpaostestapleps199aopp@zerodha.com
+Action: failed
+Status: 5.1.1
+Remote-MTA: dns; aspmx.l.google.com
+Diagnostic-Code: smtp; 550-5.1.1 The email account that you tried to reach =
+does
+    not exist. Please try 550-5.1.1 double-checking the recipient's email
+    address for typos or 550-5.1.1 unnecessary spaces. Learn more at 550 5.=
+1.1
+    https://support.google.com/mail/?p=3DNoSuchUser c95si12273060edf.464 - =
+gsmtp
+----------------------------------------------
+message/rfc822
+----------------------------------------------
+Return-Path: <kailash@nadh.in>
+Date: Sun, 23 May 2021 17:12:34 +0000
+DKIM-Signature: v=3D1; a=3Drsa-sha256; c=3Drelaxed/relaxed; d=3Dnadh.in;
+	s=3Dprotonmail2; t=3D1621789961;
+	bh=3Dquadcf+F3w95Vv4EffkoWEgj0LG46W18oiyVW2rhqjk=3D;
+	h=3DDate:To:From:Reply-To:Subject:From;
+	b=3DJjlkyS8K1NigYGOKr6a+Citz/W/NUvCj52hkQu/U5iltmNH9EgWYLJ2gB56gb7WoQ
+	 KTbJidRExtT16u2FdSHlMFpJRYiurJ3S0ko6YGZYT+FUbYqPrKC9sGrX3iHR8g3h3E
+	 0UsWl9Ny/lylN8PA70tr3IHI0ZzSYP5njITZIyJP9QfHnXK/n9r418pLtnRXoovztX
+	 797taPQIUjiVXgGDSg9AcWsRHPHGE9y0otE1gG0Vzt7kMzY/RHLq65eRvPtncy9IUU
+	 7fSbsyi8qaBIbRoFFvkJTHgusTvbyFuPfnajd+Dpm6G3xLn26ny0ZXpDwPO/ZJOsVI
+	 BbcMLTMUBJToQ=3D=3D
+To: osdktestpaostestapleps199aopp@zerodha.com
+From: Kailash Nadh <kailash@nadh.in>
+Reply-To: Kailash Nadh <kailash@nadh.in>
+Subject: Hi, this is a test!
+Message-ID: <231fe26c-de5f-1fd7-b658-6452ae74149e@nadh.in>
+MIME-Version: 1.0
+Content-Type: text/plain; charset=3Dutf-8
+Content-Transfer-Encoding: quoted-printable
+X-Spam-Status: No, score=3D-1.2 required=3D10.0 tests=3DALL_TRUSTED,DKIM_SI=
+GNED,
+	DKIM_VALID,DKIM_VALID_AU,DKIM_VALID_EF shortcircuit=3Dno
+	autolearn=3Ddisabled version=3D3.4.4
+X-Spam-Checker-Version: SpamAssassin 3.4.4 (2020-01-24) on
+	mailout.protonmail.ch
+
+Empty Message
+
+
+--ea9649a4b8e544897b542b3a1b97e2d172a7efdd385fe08531c312b947091452
+Content-Disposition: attachment; filename=email-1.3.eml
+Content-Type: message/rfc822; name=email-1.3.eml
+Content-Description: Undelivered Message
+X-Pm-Content-Encryption: on-delivery
+
+Return-Path: <kailash@nadh.in>
+Date: Sun, 23 May 2021 17:12:34 +0000
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=nadh.in;
+	s=protonmail2; t=1621789961;
+	bh=quadcf+F3w95Vv4EffkoWEgj0LG46W18oiyVW2rhqjk=;
+	h=Date:To:From:Reply-To:Subject:From;
+	b=JjlkyS8K1NigYGOKr6a+Citz/W/NUvCj52hkQu/U5iltmNH9EgWYLJ2gB56gb7WoQ
+	 KTbJidRExtT16u2FdSHlMFpJRYiurJ3S0ko6YGZYT+FUbYqPrKC9sGrX3iHR8g3h3E
+	 0UsWl9Ny/lylN8PA70tr3IHI0ZzSYP5njITZIyJP9QfHnXK/n9r418pLtnRXoovztX
+	 797taPQIUjiVXgGDSg9AcWsRHPHGE9y0otE1gG0Vzt7kMzY/RHLq65eRvPtncy9IUU
+	 7fSbsyi8qaBIbRoFFvkJTHgusTvbyFuPfnajd+Dpm6G3xLn26ny0ZXpDwPO/ZJOsVI
+	 BbcMLTMUBJToQ==
+To: osdktestpaostestapleps199aopp@zerodha.com
+From: Kailash Nadh <kailash@nadh.in>
+Reply-To: Kailash Nadh <kailash@nadh.in>
+Subject: Hi, this is a test!
+Message-ID: <231fe26c-de5f-1fd7-b658-6452ae74149e@nadh.in>
+MIME-Version: 1.0
+Content-Type: text/plain; charset=utf-8
+Content-Transfer-Encoding: quoted-printable
+X-Spam-Status: No, score=-1.2 required=10.0 tests=ALL_TRUSTED,DKIM_SIGNED,
+	DKIM_VALID,DKIM_VALID_AU,DKIM_VALID_EF shortcircuit=no
+	autolearn=disabled version=3.4.4
+X-Spam-Checker-Version: SpamAssassin 3.4.4 (2020-01-24) on
+	mailout.protonmail.ch
+
+Empty Message
+
+
+--ea9649a4b8e544897b542b3a1b97e2d172a7efdd385fe08531c312b947091452--

internal/bounce/webhooks/sendgrid.go

@@ -0,0 +1,104 @@
+package webhooks
+
+import (
+	"crypto/ecdsa"
+	"crypto/sha256"
+	"crypto/x509"
+	"encoding/asn1"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"math/big"
+	"strings"
+	"time"
+
+	"github.com/knadh/listmonk/models"
+)
+
+type sendgridNotif struct {
+	Email     string `json:"email"`
+	Timestamp int64  `json:"timestamp"`
+	Event     string `json:"event"`
+}
+
+// Sendgrid handles Sendgrid/SNS webhook notifications including confirming SNS topic subscription
+// requests and bounce notifications.
+type Sendgrid struct {
+	pubKey *ecdsa.PublicKey
+}
+
+// NewSendgrid returns a new Sendgrid instance.
+func NewSendgrid(key string) (*Sendgrid, error) {
+	// Get the certificate from the key.
+	sigB, err := base64.StdEncoding.DecodeString(key)
+	if err != nil {
+		return nil, err
+	}
+
+	pubKey, err := x509.ParsePKIXPublicKey(sigB)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Sendgrid{pubKey: pubKey.(*ecdsa.PublicKey)}, nil
+}
+
+// ProcessBounce processes Sendgrid bounce notifications and returns one or more Bounce objects.
+func (s *Sendgrid) ProcessBounce(sig, timestamp string, b []byte) ([]models.Bounce, error) {
+	if err := s.verifyNotif(sig, timestamp, b); err != nil {
+		return nil, err
+	}
+
+	var notifs []sendgridNotif
+	if err := json.Unmarshal(b, &notifs); err != nil {
+		return nil, fmt.Errorf("error unmarshalling Sendgrid notification: %v", err)
+	}
+
+	out := make([]models.Bounce, 0, len(notifs))
+	for _, n := range notifs {
+		if n.Event != "bounce" {
+			continue
+		}
+
+		tstamp := time.Unix(n.Timestamp, 0)
+		b := models.Bounce{
+			Email:     strings.ToLower(n.Email),
+			Type:      models.BounceTypeHard,
+			Meta:      json.RawMessage(b),
+			Source:    "sendgrid",
+			CreatedAt: tstamp,
+		}
+		out = append(out, b)
+	}
+
+	return out, nil
+}
+
+// verifyNotif verifies the signature on a notification payload.
+func (s *Sendgrid) verifyNotif(sig, timestamp string, b []byte) error {
+	sigB, err := base64.StdEncoding.DecodeString(sig)
+	if err != nil {
+		return err
+	}
+
+	ecdsaSig := struct {
+		R *big.Int
+		S *big.Int
+	}{}
+
+	if _, err := asn1.Unmarshal(sigB, &ecdsaSig); err != nil {
+		return fmt.Errorf("error asn1 unmarshal of signature: %v", err)
+	}
+
+	h := sha256.New()
+	h.Write([]byte(timestamp))
+	h.Write(b)
+	hash := h.Sum(nil)
+
+	if !ecdsa.Verify(s.pubKey, hash, ecdsaSig.R, ecdsaSig.S) {
+		return errors.New("invalid signature")
+	}
+
+	return nil
+}

internal/bounce/webhooks/ses.go

@@ -0,0 +1,249 @@
+package webhooks
+
+import (
+	"bytes"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/json"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/knadh/listmonk/models"
+)
+
+// AWS signature/validation logic borrowed from @cavnit's contrib:
+// https://gist.github.com/cavnit/f4d63ba52b3aa05406c07dcbca2ca6cf
+
+// https://sns.ap-southeast-1.amazonaws.com/SimpleNotificationService-010a507c1833636cd94bdb98bd93083a.pem
+var sesRegCertURL = regexp.MustCompile(`(?i)^https://sns\.[a-z0-9\-]+\.amazonaws\.com(\.cn)?/SimpleNotificationService\-[a-z0-9]+\.pem$`)
+
+// sesNotif is an individual notification wrapper posted by SNS.
+type sesNotif struct {
+	// Message may be a plaintext message or a stringified JSON payload based on the message type.
+	// Four SES messages, this is the actual payload.
+	Message string `json:"Message"`
+
+	MessageId        string `json:"MessageId"`
+	Signature        string `json:"Signature"`
+	SignatureVersion string `json:"SignatureVersion"`
+	SigningCertURL   string `json:"SigningCertURL"`
+	Subject          string `json:"Subject"`
+	Timestamp        string `json:"Timestamp"`
+	Token            string `json:"Token"`
+	TopicArn         string `json:"TopicArn"`
+	Type             string `json:"Type"`
+	SubscribeURL     string `json:"SubscribeURL"`
+	UnsubscribeURL   string `json:"UnsubscribeURL"`
+}
+
+type sesTimestamp time.Time
+
+type sesMail struct {
+	NotifType string `json:"notificationType"`
+	Bounce    struct {
+		BounceType string `json:"bounceType"`
+	} `json:"bounce"`
+	Mail struct {
+		Timestamp        sesTimestamp        `json:"timestamp"`
+		HeadersTruncated bool                `json:"headersTruncated"`
+		Destination      []string            `json:"destination"`
+		Headers          []map[string]string `json:"headers"`
+	} `json:"mail"`
+}
+
+// SES handles SES/SNS webhook notifications including confirming SNS topic subscription
+// requests and bounce notifications.
+type SES struct {
+	certs map[string]*x509.Certificate
+}
+
+// NewSES returns a new SES instance.
+func NewSES() *SES {
+	return &SES{
+		certs: make(map[string]*x509.Certificate),
+	}
+}
+
+// ProcessSubscription processes an SNS topic subscribe / unsubscribe notification
+// by parsing and verifying the payload and calling the subscribe / unsubscribe URL.
+func (s *SES) ProcessSubscription(b []byte) error {
+	var n sesNotif
+	if err := json.Unmarshal(b, &n); err != nil {
+		return fmt.Errorf("error unmarshalling SNS notification: %v", err)
+	}
+	if err := s.verifyNotif(n); err != nil {
+		return err
+	}
+
+	// Make an HTTP request to the sub/unsub URL.
+	u := n.SubscribeURL
+	if n.Type == "UnsubscriptionConfirmation" {
+		u = n.UnsubscribeURL
+	}
+
+	resp, err := http.Get(u)
+	if err != nil {
+		return fmt.Errorf("error requesting subscription URL: %v", err)
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("non 200 response on subscription URL: %v", resp.StatusCode)
+	}
+
+	return nil
+}
+
+// ProcessBounce processes an SES bounce notification and returns a Bounce object.
+func (s *SES) ProcessBounce(b []byte) (models.Bounce, error) {
+	var (
+		bounce models.Bounce
+		n      sesNotif
+	)
+	if err := json.Unmarshal(b, &n); err != nil {
+		return bounce, fmt.Errorf("error unmarshalling SES notification: %v", err)
+	}
+	if err := s.verifyNotif(n); err != nil {
+		return bounce, err
+	}
+
+	var m sesMail
+	if err := json.Unmarshal([]byte(n.Message), &m); err != nil {
+		return bounce, fmt.Errorf("error unmarshalling SES notification: %v", err)
+	}
+
+	if len(m.Mail.Destination) == 0 {
+		return bounce, errors.New("no destination e-mails found in SES notification")
+	}
+
+	typ := "soft"
+	if m.Bounce.BounceType == "Permanent" {
+		typ = "hard"
+	}
+
+	// Look for the campaign ID in headers.
+	campUUID := ""
+	if !m.Mail.HeadersTruncated {
+		for _, h := range m.Mail.Headers {
+			key, ok := h["name"]
+			if !ok || key != models.EmailHeaderCampaignUUID {
+				continue
+			}
+
+			campUUID, ok = h["value"]
+			if !ok {
+				continue
+			}
+			break
+		}
+	}
+
+	return models.Bounce{
+		Email:        strings.ToLower(m.Mail.Destination[0]),
+		CampaignUUID: campUUID,
+		Type:         typ,
+		Source:       "ses",
+		Meta:         json.RawMessage(n.Message),
+		CreatedAt:    time.Time(m.Mail.Timestamp),
+	}, nil
+}
+
+func (s *SES) buildSignature(n sesNotif) []byte {
+	var b bytes.Buffer
+	b.WriteString("Message" + "\n" + n.Message + "\n")
+	b.WriteString("MessageId" + "\n" + n.MessageId + "\n")
+
+	if n.Subject != "" {
+		b.WriteString("Subject" + "\n" + n.Subject + "\n")
+	}
+	if n.SubscribeURL != "" {
+		b.WriteString("SubscribeURL" + "\n" + n.SubscribeURL + "\n")
+	}
+
+	b.WriteString("Timestamp" + "\n" + n.Timestamp + "\n")
+
+	if n.Token != "" {
+		b.WriteString("Token" + "\n" + n.Token + "\n")
+	}
+	b.WriteString("TopicArn" + "\n" + n.TopicArn + "\n")
+	b.WriteString("Type" + "\n" + n.Type + "\n")
+
+	return b.Bytes()
+}
+
+// verifyNotif verifies the signature on a notification payload.
+func (s *SES) verifyNotif(n sesNotif) error {
+	// Get the message signing certificate.
+	cert, err := s.getCert(n.SigningCertURL)
+	if err != nil {
+		return fmt.Errorf("error getting SNS cert: %v", err)
+	}
+
+	sign, err := base64.StdEncoding.DecodeString(n.Signature)
+	if err != nil {
+		return err
+	}
+
+	return cert.CheckSignature(x509.SHA1WithRSA, s.buildSignature(n), sign)
+}
+
+// getCert takes the SNS certificate URL and fetches it and caches it for the first time,
+// and returns the cached cert for subsequent calls.
+func (s *SES) getCert(certURL string) (*x509.Certificate, error) {
+	// Ensure that the cert URL is Amazon's.
+	u, err := url.Parse(certURL)
+	if err != nil {
+		return nil, err
+	}
+	if !sesRegCertURL.MatchString(certURL) {
+		return nil, fmt.Errorf("invalid SNS certificate URL: %v", u.Host)
+	}
+
+	// Return if it's cached.
+	if c, ok := s.certs[u.Path]; ok {
+		return c, nil
+	}
+
+	// Fetch the certificate.
+	resp, err := http.Get(certURL)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("invalid SNS certificate URL: %v", u.Host)
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	p, _ := pem.Decode(body)
+	if p == nil {
+		return nil, errors.New("invalid PEM")
+	}
+
+	cert, err := x509.ParseCertificate(p.Bytes)
+
+	// Cache the cert in-memory.
+	s.certs[u.Path] = cert
+
+	return cert, err
+}
+
+func (st *sesTimestamp) UnmarshalJSON(b []byte) error {
+	t, err := time.Parse("2006-01-02T15:04:05.999999999Z", strings.Trim(string(b), `"`))
+	if err != nil {
+		return err
+	}
+	*st = sesTimestamp(t)
+	return nil
+}

internal/manager/manager.go

@@ -21,27 +21,37 @@ const (
 	// BaseTPL is the name of the base template.
 	BaseTPL = "base"
 
+	BounceTypeBlocklist = "blocklist"
+	BounceTypeDelete    = "delete"
+
 	// ContentTpl is the name of the compiled message.
 	ContentTpl = "content"
 
 	dummyUUID = "00000000-0000-0000-0000-000000000000"
 )
 
-// DataSource represents a data backend, such as a database,
+// Store represents a data backend, such as a database,
 // that provides subscriber and campaign records.
-type DataSource interface {
+type Store interface {
 	NextCampaigns(excludeIDs []int64) ([]*models.Campaign, error)
 	NextSubscribers(campID, limit int) ([]models.Subscriber, error)
 	GetCampaign(campID int) (*models.Campaign, error)
 	UpdateCampaignStatus(campID int, status string) error
 	CreateLink(url string) (string, error)
+
+	// RecordBounce records an external bounce event identified by
+	// a user's UUID/e-mail and a campaign UUID.
+	RecordBounce(b models.Bounce) (int64, int, error)
+
+	BlocklistSubscriber(id int64) error
+	DeleteSubscriber(id int64) error
 }
 
 // Manager handles the scheduling, processing, and queuing of campaigns
 // and message pushes.
 type Manager struct {
 	cfg        Config
-	src        DataSource
+	store      Store
 	i18n       *i18n.I18n
 	messengers map[string]messenger.Messenger
 	notifCB    models.AdminNotifCallback
@@ -62,6 +72,7 @@ type Manager struct {
 	campMsgErrorQueue  chan msgError
 	campMsgErrorCounts map[int]int
 	msgQueue           chan Message
+	bounceQueue        chan models.Bounce
 
 	// Sliding window keeps track of the total number of messages sent in a period
 	// and on reaching the specified limit, waits until the window is over before
@@ -113,6 +124,8 @@ type Config struct {
 	MessageURL            string
 	ViewTrackURL          string
 	UnsubHeader           bool
+	BounceCount           int
+	BounceAction          string
 }
 
 type msgError struct {
@@ -120,8 +133,10 @@ type msgError struct {
 	err  error
 }
 
+var pushTimeout = time.Second * 3
+
 // New returns a new instance of Mailer.
-func New(cfg Config, src DataSource, notifCB models.AdminNotifCallback, i *i18n.I18n, l *log.Logger) *Manager {
+func New(cfg Config, store Store, notifCB models.AdminNotifCallback, i *i18n.I18n, l *log.Logger) *Manager {
 	if cfg.BatchSize < 1 {
 		cfg.BatchSize = 1000
 	}
@@ -134,7 +149,7 @@ func New(cfg Config, src DataSource, notifCB models.AdminNotifCallback, i *i18n.
 
 	return &Manager{
 		cfg:                cfg,
-		src:                src,
+		store:              store,
 		i18n:               i,
 		notifCB:            notifCB,
 		logger:             l,
@@ -144,6 +159,7 @@ func New(cfg Config, src DataSource, notifCB models.AdminNotifCallback, i *i18n.
 		subFetchQueue:      make(chan *models.Campaign, cfg.Concurrency),
 		campMsgQueue:       make(chan CampaignMessage, cfg.Concurrency*2),
 		msgQueue:           make(chan Message, cfg.Concurrency),
+		bounceQueue:        make(chan models.Bounce, cfg.Concurrency),
 		campMsgErrorQueue:  make(chan msgError, cfg.MaxSendErrors),
 		campMsgErrorCounts: make(map[int]int),
 		slidingWindowStart: time.Now(),
@@ -184,7 +200,7 @@ func (m *Manager) AddMessenger(msg messenger.Messenger) error {
 // PushMessage pushes an arbitrary non-campaign Message to be sent out by the workers.
 // It times out if the queue is busy.
 func (m *Manager) PushMessage(msg Message) error {
-	t := time.NewTicker(time.Second * 3)
+	t := time.NewTicker(pushTimeout)
 	defer t.Stop()
 
 	select {
@@ -199,7 +215,7 @@ func (m *Manager) PushMessage(msg Message) error {
 // PushCampaignMessage pushes a campaign messages to be sent out by the workers.
 // It times out if the queue is busy.
 func (m *Manager) PushCampaignMessage(msg CampaignMessage) error {
-	t := time.NewTicker(time.Second * 3)
+	t := time.NewTicker(pushTimeout)
 	defer t.Stop()
 
 	select {
@@ -224,6 +240,20 @@ func (m *Manager) HasRunningCampaigns() bool {
 	return len(m.camps) > 0
 }
 
+// PushBounce records a bounce event.
+func (m *Manager) PushBounce(b models.Bounce) error {
+	t := time.NewTicker(pushTimeout)
+	defer t.Stop()
+
+	select {
+	case m.bounceQueue <- b:
+	case <-t.C:
+		m.logger.Printf("bounce pushed timed out: %s / %s", b.SubscriberUUID, b.Email)
+		return errors.New("bounce push timed out")
+	}
+	return nil
+}
+
 // Run is a blocking function (that should be invoked as a goroutine)
 // that scans the data source at regular intervals for pending campaigns,
 // and queues them for processing. The process queue fetches batches of
@@ -235,7 +265,7 @@ func (m *Manager) Run(tick time.Duration) {
 
 	// Spawn N message workers.
 	for i := 0; i < m.cfg.Concurrency; i++ {
-		go m.messageWorker()
+		go m.worker()
 	}
 
 	// Fetch the next set of subscribers for a campaign and process them.
@@ -262,9 +292,9 @@ func (m *Manager) Run(tick time.Duration) {
 	}
 }
 
-// messageWorker is a blocking function that listens to the message queue
+// worker is a blocking function that perpetually listents to events (message, bounce) on different
-// and pushes out incoming messages on it to the messenger.
+// queues and processes them.
-func (m *Manager) messageWorker() {
+func (m *Manager) worker() {
 	// Counter to keep track of the message / sec rate limit.
 	numMsg := 0
 	for {
@@ -294,14 +324,18 @@ func (m *Manager) messageWorker() {
 				Campaign:    msg.Campaign,
 			}
 
+			h := textproto.MIMEHeader{}
+			h.Set(models.EmailHeaderCampaignUUID, msg.Campaign.UUID)
+			h.Set(models.EmailHeaderSubscriberUUID, msg.Subscriber.UUID)
+
 			// Attach List-Unsubscribe headers?
 			if m.cfg.UnsubHeader {
-				h := textproto.MIMEHeader{}
 				h.Set("List-Unsubscribe-Post", "List-Unsubscribe=One-Click")
 				h.Set("List-Unsubscribe", `<`+msg.unsubURL+`>`)
-				out.Headers = h
 			}
 
+			out.Headers = h
+
 			if err := m.messengers[msg.Campaign.Messenger].Push(out); err != nil {
 				m.logger.Printf("error sending message in campaign %s: subscriber %s: %v",
 					msg.Campaign.Name, msg.Subscriber.UUID, err)
@@ -331,6 +365,30 @@ func (m *Manager) messageWorker() {
 			if err != nil {
 				m.logger.Printf("error sending message '%s': %v", msg.Subject, err)
 			}
+
+		// Bounce event.
+		case b, ok := <-m.bounceQueue:
+			if !ok {
+				return
+			}
+
+			subID, count, err := m.store.RecordBounce(b)
+			if err != nil {
+				m.logger.Printf("error recording bounce %s / %s", b.SubscriberUUID, b.Email)
+			}
+
+			if count >= m.cfg.BounceCount {
+				switch m.cfg.BounceAction {
+				case BounceTypeBlocklist:
+					err = m.store.BlocklistSubscriber(subID)
+				case BounceTypeDelete:
+					err = m.store.DeleteSubscriber(subID)
+				}
+
+				if err != nil {
+					m.logger.Printf("error executing bounce for subscriber: %s", b.SubscriberUUID)
+				}
+			}
 		}
 	}
 }
@@ -403,7 +461,7 @@ func (m *Manager) scanCampaigns(tick time.Duration) {
 		select {
 		// Periodically scan the data source for campaigns to process.
 		case <-t.C:
-			campaigns, err := m.src.NextCampaigns(m.getPendingCampaignIDs())
+			campaigns, err := m.store.NextCampaigns(m.getPendingCampaignIDs())
 			if err != nil {
 				m.logger.Printf("error fetching campaigns: %v", err)
 				continue
@@ -457,7 +515,7 @@ func (m *Manager) scanCampaigns(tick time.Duration) {
 func (m *Manager) addCampaign(c *models.Campaign) error {
 	// Validate messenger.
 	if _, ok := m.messengers[c.Messenger]; !ok {
-		m.src.UpdateCampaignStatus(c.ID, models.CampaignStatusCancelled)
+		m.store.UpdateCampaignStatus(c.ID, models.CampaignStatusCancelled)
 		return fmt.Errorf("unknown messenger %s on campaign %s", c.Messenger, c.Name)
 	}
 
@@ -491,7 +549,7 @@ func (m *Manager) getPendingCampaignIDs() []int64 {
 // have been processed, or that a campaign has been paused or cancelled.
 func (m *Manager) nextSubscribers(c *models.Campaign, batchSize int) (bool, error) {
 	// Fetch a batch of subscribers.
-	subs, err := m.src.NextSubscribers(c.ID, batchSize)
+	subs, err := m.store.NextSubscribers(c.ID, batchSize)
 	if err != nil {
 		return false, fmt.Errorf("error fetching campaign subscribers (%s): %v", c.Name, err)
 	}
@@ -566,7 +624,7 @@ func (m *Manager) exhaustCampaign(c *models.Campaign, status string) (*models.Ca
 	// A status has been passed. Change the campaign's status
 	// without further checks.
 	if status != "" {
-		if err := m.src.UpdateCampaignStatus(c.ID, status); err != nil {
+		if err := m.store.UpdateCampaignStatus(c.ID, status); err != nil {
 			m.logger.Printf("error updating campaign (%s) status to %s: %v", c.Name, status, err)
 		} else {
 			m.logger.Printf("set campaign (%s) to %s", c.Name, status)
@@ -575,7 +633,7 @@ func (m *Manager) exhaustCampaign(c *models.Campaign, status string) (*models.Ca
 	}
 
 	// Fetch the up-to-date campaign status from the source.
-	cm, err := m.src.GetCampaign(c.ID)
+	cm, err := m.store.GetCampaign(c.ID)
 	if err != nil {
 		return nil, err
 	}
@@ -583,7 +641,7 @@ func (m *Manager) exhaustCampaign(c *models.Campaign, status string) (*models.Ca
 	// If a running campaign has exhausted subscribers, it's finished.
 	if cm.Status == models.CampaignStatusRunning {
 		cm.Status = models.CampaignStatusFinished
-		if err := m.src.UpdateCampaignStatus(c.ID, models.CampaignStatusFinished); err != nil {
+		if err := m.store.UpdateCampaignStatus(c.ID, models.CampaignStatusFinished); err != nil {
 			m.logger.Printf("error finishing campaign (%s): %v", c.Name, err)
 		} else {
 			m.logger.Printf("campaign (%s) finished", c.Name)
@@ -606,7 +664,7 @@ func (m *Manager) trackLink(url, campUUID, subUUID string) string {
 	m.linksMut.RUnlock()
 
 	// Register link.
-	uu, err := m.src.CreateLink(url)
+	uu, err := m.store.CreateLink(url)
 	if err != nil {
 		m.logger.Printf("error registering tracking for link '%s': %v", url, err)
 

internal/migrations/v2.0.0.go

@@ -0,0 +1,43 @@
+package migrations
+
+import (
+	"github.com/jmoiron/sqlx"
+	"github.com/knadh/koanf"
+	"github.com/knadh/stuffbin"
+)
+
+// V2_0_0 performs the DB migrations for v.1.0.0.
+func V2_0_0(db *sqlx.DB, fs stuffbin.FileSystem, ko *koanf.Koanf) error {
+	if _, err := db.Exec(`
+		CREATE TABLE IF NOT EXISTS bounces (
+		    id               SERIAL PRIMARY KEY,
+		    subscriber_id    INTEGER NOT NULL REFERENCES subscribers(id) ON DELETE CASCADE ON UPDATE CASCADE,
+		    campaign_id      INTEGER NULL REFERENCES campaigns(id) ON DELETE SET NULL ON UPDATE CASCADE,
+		    type             bounce_type NOT NULL DEFAULT 'hard',
+		    source           TEXT NOT NULL DEFAULT '',
+		    meta             JSONB NOT NULL DEFAULT '{}',
+		    created_at       TIMESTAMP WITH TIME ZONE DEFAULT NOW()
+		);
+		CREATE INDEX IF NOT EXISTS idx_bounces_sub_id ON bounces(subscriber_id);
+		CREATE INDEX IF NOT EXISTS idx_bounces_camp_id ON bounces(campaign_id);
+		CREATE INDEX IF NOT EXISTS idx_bounces_source ON bounces(source);
+	`); err != nil {
+		return err
+	}
+
+	if _, err := db.Exec(`
+		INSERT INTO settings (key, value) SELECT k, v::JSONB FROM (VALUES
+	    ('bounce.enabled', 'false'),
+	    ('bounce.webhooks_enabled', 'false'),
+	    ('bounce.count', '2'),
+	    ('bounce.action', '"blocklist"'),
+	    ('bounce.ses_enabled', 'false'),
+	    ('bounce.sendgrid_enabled', 'false'),
+	    ('bounce.sendgrid_key', '""'),
+	    ('bounce.mailboxes', '[{"enabled":false, "type": "pop", "host":"pop.yoursite.com","port":995,"auth_protocol":"userpass","username":"username","password":"password","return_path": "bounce@listmonk.yoursite.com","scan_interval":"15m","tls_enabled":true,"tls_skip_verify":false}]'))
+	    VALS (k, v) WHERE NOT EXISTS(SELECT * FROM settings LIMIT 1);`); err != nil {
+		return err
+	}
+
+	return nil
+}

models/models.go

@@ -9,6 +9,7 @@ import (
 	"html/template"
 	"regexp"
 	"strings"
+	"time"
 
 	"github.com/jmoiron/sqlx"
 	"github.com/jmoiron/sqlx/types"
@@ -62,6 +63,13 @@ const (
 
 	// ContentTpl is the name of the compiled message.
 	ContentTpl = "content"
+
+	// Headers attached to e-mails for bounce tracking.
+	EmailHeaderSubscriberUUID = "X-Listmonk-Subscriber"
+	EmailHeaderCampaignUUID   = "X-Listmonk-Campaign"
+
+	BounceTypeHard = "hard"
+	BounceTypeSoft = "soft"
 )
 
 // regTplFunc represents contains a regular expression for wrapping and
@@ -201,6 +209,7 @@ type CampaignMeta struct {
 	CampaignID int `db:"campaign_id" json:"-"`
 	Views      int `db:"views" json:"views"`
 	Clicks     int `db:"clicks" json:"clicks"`
+	Bounces    int `db:"bounces" json:"bounces"`
 
 	// This is a list of {list_id, name} pairs unlike Subscriber.Lists[]
 	// because lists can be deleted after a campaign is finished, resulting
@@ -226,6 +235,27 @@ type Template struct {
 	IsDefault bool   `db:"is_default" json:"is_default"`
 }
 
+// Bounce represents a single bounce event.
+type Bounce struct {
+	ID        int             `db:"id" json:"id"`
+	Type      string          `db:"type" json:"type"`
+	Source    string          `db:"source" json:"source"`
+	Meta      json.RawMessage `db:"meta" json:"meta"`
+	CreatedAt time.Time       `db:"created_at" json:"created_at"`
+
+	// One of these should be provided.
+	Email          string `db:"email" json:"email,omitempty"`
+	SubscriberUUID string `db:"subscriber_uuid" json:"subscriber_uuid,omitempty"`
+	SubscriberID   int    `db:"subscriber_id" json:"subscriber_id,omitempty"`
+
+	CampaignUUID string           `db:"campaign_uuid" json:"campaign_uuid,omitempty"`
+	Campaign     *json.RawMessage `db:"campaign" json:"campaign"`
+
+	// Pseudofield for getting the total number of bounces
+	// in searches and queries.
+	Total int `db:"total" json:"-"`
+}
+
 // markdown is a global instance of Markdown parser and renderer.
 var markdown = goldmark.New(
 	goldmark.WithRendererOptions(
@@ -310,6 +340,7 @@ func (camps Campaigns) LoadStats(stmt *sqlx.Stmt) error {
 			camps[i].Lists = c.Lists
 			camps[i].Views = c.Views
 			camps[i].Clicks = c.Clicks
+			camps[i].Bounces = c.Bounces
 		}
 	}
 

queries.sql

@@ -451,15 +451,22 @@ clicks AS (
     SELECT campaign_id, COUNT(campaign_id) as num FROM link_clicks
     WHERE campaign_id = ANY($1)
     GROUP BY campaign_id
+),
+bounces AS (
+    SELECT campaign_id, COUNT(campaign_id) as num FROM bounces
+    WHERE campaign_id = ANY($1)
+    GROUP BY campaign_id
 )
 SELECT id as campaign_id,
     COALESCE(v.num, 0) AS views,
     COALESCE(c.num, 0) AS clicks,
+    COALESCE(b.num, 0) AS bounces,
     COALESCE(l.lists, '[]') AS lists
 FROM (SELECT id FROM UNNEST($1) AS id) x
 LEFT JOIN lists AS l ON (l.campaign_id = id)
 LEFT JOIN views AS v ON (v.campaign_id = id)
 LEFT JOIN clicks AS c ON (c.campaign_id = id)
+LEFT JOIN bounces AS b ON (b.campaign_id = id)
 ORDER BY ARRAY_POSITION($1, id);
 
 -- name: get-campaign-for-preview
@@ -781,3 +788,67 @@ SELECT JSON_OBJECT_AGG(key, value) AS settings
 UPDATE settings AS s SET value = c.value
     -- For each key in the incoming JSON map, update the row with the key and its value.
     FROM(SELECT * FROM JSONB_EACH($1)) AS c(key, value) WHERE s.key = c.key;
+
+-- name: record-bounce
+-- Insert a bounce and count the bounces for the subscriber and either unsubscribe them,
+WITH sub AS (
+    SELECT id, status FROM subscribers WHERE CASE WHEN $1 != '' THEN uuid = $1::UUID ELSE email = $2 END
+),
+camp AS (
+    SELECT id FROM campaigns WHERE $3 != '' AND uuid = $3::UUID
+),
+bounce AS (
+    -- Record the bounce if it the subscriber is not already blocklisted;
+    INSERT INTO bounces (subscriber_id, campaign_id, type, source, meta, created_at)
+    SELECT (SELECT id FROM sub), (SELECT id FROM camp), $4, $5, $6, $7
+    WHERE NOT EXISTS (SELECT 1 WHERE (SELECT status FROM sub) = 'blocklisted')
+),
+num AS (
+    SELECT COUNT(*) AS num FROM bounces WHERE subscriber_id = (SELECT id FROM sub)
+),
+-- block1 and block2 will run when $8 = 'blocklist' and the number of bounces exceed $8.
+block1 AS (
+    UPDATE subscribers SET status='blocklisted'
+    WHERE $9 = 'blocklist' AND (SELECT num FROM num) >= $8 AND id = (SELECT id FROM sub) AND (SELECT status FROM sub) != 'blocklisted'
+),
+block2 AS (
+    UPDATE subscriber_lists SET status='unsubscribed'
+    WHERE $9 = 'blocklist' AND (SELECT num FROM num) >= $8 AND subscriber_id = (SELECT id FROM sub) AND (SELECT status FROM sub) != 'blocklisted'
+)
+-- This delete  will only run when $9 = 'delete' and the number of bounces exceed $8.
+DELETE FROM subscribers
+    WHERE $9 = 'delete' AND (SELECT num FROM num) >= $8 AND id = (SELECT id FROM sub);
+
+-- name: query-bounces
+SELECT COUNT(*) OVER () AS total,
+    bounces.id,
+    bounces.type,
+    bounces.source,
+    bounces.meta,
+    bounces.created_at,
+    bounces.subscriber_id,
+    subscribers.uuid AS subscriber_uuid,
+    subscribers.email AS email,
+    subscribers.email AS email,
+    (
+        CASE WHEN bounces.campaign_id IS NOT NULL
+        THEN JSON_BUILD_OBJECT('id', bounces.campaign_id, 'name', campaigns.name)
+        ELSE NULL END
+    ) AS campaign
+FROM bounces
+LEFT JOIN subscribers ON (subscribers.id = bounces.subscriber_id)
+LEFT JOIN campaigns ON (campaigns.id = bounces.campaign_id)
+WHERE ($1 = 0 OR bounces.id = $1)
+    AND ($2 = 0 OR bounces.campaign_id = $2)
+    AND ($3 = 0 OR bounces.subscriber_id = $3)
+    AND ($4 = '' OR bounces.source = $4)
+ORDER BY %s %s OFFSET $5 LIMIT $6;
+
+-- name: delete-bounces
+DELETE FROM bounces WHERE ARRAY_LENGTH($1::INT[], 1) IS NULL OR id = ANY($1);
+
+-- name: delete-bounces-by-subscriber
+WITH sub AS (
+    SELECT id FROM subscribers WHERE CASE WHEN $1 > 0 THEN id = $1 ELSE uuid = $2 END
+)
+DELETE FROM bounces WHERE subscriber_id = (SELECT id FROM sub);

schema.sql

@@ -5,6 +5,7 @@ DROP TYPE IF EXISTS subscription_status CASCADE; CREATE TYPE subscription_status
 DROP TYPE IF EXISTS campaign_status CASCADE; CREATE TYPE campaign_status AS ENUM ('draft', 'running', 'scheduled', 'paused', 'cancelled', 'finished');
 DROP TYPE IF EXISTS campaign_type CASCADE; CREATE TYPE campaign_type AS ENUM ('regular', 'optin');
 DROP TYPE IF EXISTS content_type CASCADE; CREATE TYPE content_type AS ENUM ('richtext', 'html', 'plain', 'markdown');
+DROP TYPE IF EXISTS bounce_type CASCADE; CREATE TYPE bounce_type AS ENUM ('soft', 'hard', 'complaint');
 
 -- subscribers
 DROP TABLE IF EXISTS subscribers CASCADE;
@@ -201,4 +202,28 @@ INSERT INTO settings (key, value) VALUES
     ('smtp',
         '[{"enabled":true, "host":"smtp.yoursite.com","port":25,"auth_protocol":"cram","username":"username","password":"password","hello_hostname":"","max_conns":10,"idle_timeout":"15s","wait_timeout":"5s","max_msg_retries":2,"tls_enabled":true,"tls_skip_verify":false,"email_headers":[]},
           {"enabled":false, "host":"smtp2.yoursite.com","port":587,"auth_protocol":"plain","username":"username","password":"password","hello_hostname":"","max_conns":10,"idle_timeout":"15s","wait_timeout":"5s","max_msg_retries":2,"tls_enabled":false,"tls_skip_verify":false,"email_headers":[]}]'),
-    ('messengers', '[]');
+    ('messengers', '[]'),
+    ('bounce.enabled', 'false'),
+    ('bounce.webhooks_enabled', 'false'),
+    ('bounce.count', '2'),
+    ('bounce.action', '"blocklist"'),
+    ('bounce.ses_enabled', 'false'),
+    ('bounce.sendgrid_enabled', 'false'),
+    ('bounce.sendgrid_key', '""'),
+    ('bounce.mailboxes',
+        '[{"enabled":false, "type": "pop", "host":"pop.yoursite.com","port":995,"auth_protocol":"userpass","username":"username","password":"password","return_path": "bounce@listmonk.yoursite.com","scan_interval":"15m","tls_enabled":true,"tls_skip_verify":false}]');
+
+-- bounces
+DROP TABLE IF EXISTS bounces CASCADE;
+CREATE TABLE bounces (
+    id               SERIAL PRIMARY KEY,
+    subscriber_id    INTEGER NOT NULL REFERENCES subscribers(id) ON DELETE CASCADE ON UPDATE CASCADE,
+    campaign_id      INTEGER NULL REFERENCES campaigns(id) ON DELETE SET NULL ON UPDATE CASCADE,
+    type             bounce_type NOT NULL DEFAULT 'hard',
+    source           TEXT NOT NULL DEFAULT '',
+    meta             JSONB NOT NULL DEFAULT '{}',
+    created_at       TIMESTAMP WITH TIME ZONE DEFAULT NOW()
+);
+DROP INDEX IF EXISTS idx_bounces_sub_id; CREATE INDEX idx_bounces_sub_id ON bounces(subscriber_id);
+DROP INDEX IF EXISTS idx_bounces_camp_id; CREATE INDEX idx_bounces_camp_id ON bounces(campaign_id);
+DROP INDEX IF EXISTS idx_bounces_source; CREATE INDEX idx_bounces_source ON bounces(source);