Add support for file attachments on campaigns (#1341)

- Adds support for arbitrary file uploads with an admin setting to select allowed file extensions.
- Adds support for attaching media (files) to campaigns.
This commit is contained in:
Kailash Nadh 2023-05-18 16:55:59 +05:30 committed by GitHub
parent cb2a579252
commit 3b9a0f782e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
27 changed files with 478 additions and 140 deletions

View file

@ -32,6 +32,8 @@ type campaignReq struct {
// to the outside world.
ListIDs []int `json:"lists"`
MediaIDs []int `json:"media"`
// This is only relevant to campaign test requests.
SubscriberEmails pq.StringArray `json:"subscribers"`
}
@ -220,7 +222,7 @@ func handleCreateCampaign(c echo.Context) error {
o.ArchiveTemplateID = o.TemplateID
}
out, err := app.core.CreateCampaign(o.Campaign, o.ListIDs)
out, err := app.core.CreateCampaign(o.Campaign, o.ListIDs, o.MediaIDs)
if err != nil {
return err
}
@ -264,7 +266,7 @@ func handleUpdateCampaign(c echo.Context) error {
o = c
}
out, err := app.core.UpdateCampaign(id, o.Campaign, o.ListIDs, o.SendLater)
out, err := app.core.UpdateCampaign(id, o.Campaign, o.ListIDs, o.MediaIDs, o.SendLater)
if err != nil {
return err
}
@ -437,6 +439,11 @@ func handleTestCampaign(c echo.Context) error {
camp.ContentType = req.ContentType
camp.Headers = req.Headers
camp.TemplateID = req.TemplateID
for _, id := range req.MediaIDs {
if id > 0 {
camp.MediaIDs = append(camp.MediaIDs, int64(id))
}
}
// Send the test messages.
for _, s := range subs {

View file

@ -92,7 +92,11 @@ type constants struct {
OptinURL string
MessageURL string
ArchiveURL string
MediaProvider string
MediaUpload struct {
Provider string
Extensions []string
}
BounceWebhooksEnabled bool
BounceSESEnabled bool
@ -368,7 +372,8 @@ func initConstants() *constants {
c.RootURL = strings.TrimRight(c.RootURL, "/")
c.Lang = ko.String("app.lang")
c.Privacy.Exportable = maps.StringSliceToLookupMap(ko.Strings("privacy.exportable"))
c.MediaProvider = ko.String("upload.provider")
c.MediaUpload.Provider = ko.String("upload.provider")
c.MediaUpload.Extensions = ko.Strings("upload.extensions")
c.Privacy.DomainBlocklist = ko.Strings("privacy.domain_blocklist")
// Static URLS.
@ -448,7 +453,7 @@ func initCampaignManager(q *models.Queries, cs *constants, app *App) *manager.Ma
SlidingWindowRate: ko.Int("app.message_sliding_window_rate"),
ScanInterval: time.Second * 5,
ScanCampaigns: !ko.Bool("passive"),
}, newManagerStore(q), campNotifCB, app.i18n, lo)
}, newManagerStore(q, app.core, app.media), campNotifCB, app.i18n, lo)
}
func initTxTemplates(m *manager.Manager, app *App) {

View file

@ -161,6 +161,7 @@ func install(lastVer string, db *sqlx.DB, fs stuffbin.FileSystem, prompt, idempo
false,
archiveTplID,
`{"name": "Subscriber"}`,
nil,
); err != nil {
lo.Fatalf("error creating sample campaign: %v", err)
}

View file

@ -1,27 +1,37 @@
package main
import (
"net/http"
"github.com/gofrs/uuid"
"github.com/knadh/listmonk/internal/core"
"github.com/knadh/listmonk/internal/manager"
"github.com/knadh/listmonk/internal/media"
"github.com/knadh/listmonk/models"
"github.com/lib/pq"
)
// runnerDB implements runner.DataSource over the primary
// store implements DataSource over the primary
// database.
type runnerDB struct {
type store struct {
queries *models.Queries
core *core.Core
media media.Store
h *http.Client
}
func newManagerStore(q *models.Queries) *runnerDB {
return &runnerDB{
func newManagerStore(q *models.Queries, c *core.Core, m media.Store) *store {
return &store{
queries: q,
core: c,
media: m,
}
}
// NextCampaigns retrieves active campaigns ready to be processed.
func (r *runnerDB) NextCampaigns(excludeIDs []int64) ([]*models.Campaign, error) {
func (s *store) NextCampaigns(excludeIDs []int64) ([]*models.Campaign, error) {
var out []*models.Campaign
err := r.queries.NextCampaigns.Select(&out, pq.Int64Array(excludeIDs))
err := s.queries.NextCampaigns.Select(&out, pq.Int64Array(excludeIDs))
return out, err
}
@ -29,27 +39,46 @@ func (r *runnerDB) NextCampaigns(excludeIDs []int64) ([]*models.Campaign, error)
// Since batches are processed sequentially, the retrieval is ordered by ID,
// and every batch takes the last ID of the last batch and fetches the next
// batch above that.
func (r *runnerDB) NextSubscribers(campID, limit int) ([]models.Subscriber, error) {
func (s *store) NextSubscribers(campID, limit int) ([]models.Subscriber, error) {
var out []models.Subscriber
err := r.queries.NextCampaignSubscribers.Select(&out, campID, limit)
err := s.queries.NextCampaignSubscribers.Select(&out, campID, limit)
return out, err
}
// GetCampaign fetches a campaign from the database.
func (r *runnerDB) GetCampaign(campID int) (*models.Campaign, error) {
func (s *store) GetCampaign(campID int) (*models.Campaign, error) {
var out = &models.Campaign{}
err := r.queries.GetCampaign.Get(out, campID, nil, "default")
err := s.queries.GetCampaign.Get(out, campID, nil, "default")
return out, err
}
// UpdateCampaignStatus updates a campaign's status.
func (r *runnerDB) UpdateCampaignStatus(campID int, status string) error {
_, err := r.queries.UpdateCampaignStatus.Exec(campID, status)
func (s *store) UpdateCampaignStatus(campID int, status string) error {
_, err := s.queries.UpdateCampaignStatus.Exec(campID, status)
return err
}
// GetAttachment fetches a media attachment blob.
func (s *store) GetAttachment(mediaID int) (models.Attachment, error) {
m, err := s.core.GetMedia(mediaID, "", s.media)
if err != nil {
return models.Attachment{}, err
}
b, err := s.media.GetBlob(m.URL)
if err != nil {
return models.Attachment{}, err
}
return models.Attachment{
Name: m.Filename,
Content: b,
Header: manager.MakeAttachmentHeader(m.Filename, "base64", m.ContentType),
}, nil
}
// CreateLink registers a URL with a UUID for tracking clicks and returns the UUID.
func (r *runnerDB) CreateLink(url string) (string, error) {
func (s *store) CreateLink(url string) (string, error) {
// Create a new UUID for the URL. If the URL already exists in the DB
// the UUID in the database is returned.
uu, err := uuid.NewV4()
@ -58,7 +87,7 @@ func (r *runnerDB) CreateLink(url string) (string, error) {
}
var out string
if err := r.queries.CreateLink.Get(&out, uu, url); err != nil {
if err := s.queries.CreateLink.Get(&out, uu, url); err != nil {
return "", err
}
@ -66,13 +95,13 @@ func (r *runnerDB) CreateLink(url string) (string, error) {
}
// RecordBounce records a bounce event and returns the bounce count.
func (r *runnerDB) RecordBounce(b models.Bounce) (int64, int, error) {
func (s *store) RecordBounce(b models.Bounce) (int64, int, error) {
var res = struct {
SubscriberID int64 `db:"subscriber_id"`
Num int `db:"num"`
}{}
err := r.queries.UpdateCampaignStatus.Select(&res,
err := s.queries.UpdateCampaignStatus.Select(&res,
b.SubscriberUUID,
b.Email,
b.CampaignUUID,
@ -83,12 +112,12 @@ func (r *runnerDB) RecordBounce(b models.Bounce) (int64, int, error) {
return res.SubscriberID, res.Num, err
}
func (r *runnerDB) BlocklistSubscriber(id int64) error {
_, err := r.queries.BlocklistSubscribers.Exec(pq.Int64Array{id})
func (s *store) BlocklistSubscriber(id int64) error {
_, err := s.queries.BlocklistSubscribers.Exec(pq.Int64Array{id})
return err
}
func (r *runnerDB) DeleteSubscriber(id int64) error {
_, err := r.queries.DeleteSubscribers.Exec(pq.Int64Array{id})
func (s *store) DeleteSubscriber(id int64) error {
_, err := s.queries.DeleteSubscribers.Exec(pq.Int64Array{id})
return err
}

View file

@ -15,16 +15,12 @@ import (
const (
thumbPrefix = "thumb_"
thumbnailSize = 120
thumbnailSize = 250
)
// validMimes is the list of image types allowed to be uploaded.
var (
validMimes = []string{"image/jpg", "image/jpeg", "image/png", "image/gif", "image/svg+xml"}
validExts = []string{".jpg", ".jpeg", ".png", ".gif", ".svg"}
// Vector extensions that don't need to be resized for thumbnails.
vectorExts = []string{".svg"}
vectorExts = []string{"svg"}
imageExts = []string{"gif", "png", "jpg", "jpeg"}
)
// handleUploadMedia handles media file uploads.
@ -39,23 +35,6 @@ func handleUploadMedia(c echo.Context) error {
app.i18n.Ts("media.invalidFile", "error", err.Error()))
}
// Validate file extension.
ext := strings.ToLower(filepath.Ext(file.Filename))
if ok := inArray(ext, validExts); !ok {
return echo.NewHTTPError(http.StatusBadRequest,
app.i18n.Ts("media.unsupportedFileType", "type", ext))
}
// Validate file's mime.
typ := file.Header.Get("Content-type")
if ok := inArray(typ, validMimes); !ok {
return echo.NewHTTPError(http.StatusBadRequest,
app.i18n.Ts("media.unsupportedFileType", "type", typ))
}
// Generate filename
fName := makeFilename(file.Filename)
// Read file contents in memory
src, err := file.Open()
if err != nil {
@ -64,8 +43,23 @@ func handleUploadMedia(c echo.Context) error {
}
defer src.Close()
var (
// Naive check for content type and extension.
ext = strings.TrimPrefix(strings.ToLower(filepath.Ext(file.Filename)), ".")
contentType = file.Header.Get("Content-Type")
)
// Validate file extension.
if !inArray("*", app.constants.MediaUpload.Extensions) {
if ok := inArray(ext, app.constants.MediaUpload.Extensions); !ok {
return echo.NewHTTPError(http.StatusBadRequest,
app.i18n.Ts("media.unsupportedFileType", "type", ext))
}
}
// Upload the file.
fName, err = app.media.Put(fName, typ, src)
fName := makeFilename(file.Filename)
fName, err = app.media.Put(fName, contentType, src)
if err != nil {
app.log.Printf("error uploading file: %v", err)
cleanUp = true
@ -73,22 +67,26 @@ func handleUploadMedia(c echo.Context) error {
app.i18n.Ts("media.errorUploading", "error", err.Error()))
}
var (
thumbfName = ""
width = 0
height = 0
)
defer func() {
// If any of the subroutines in this function fail,
// the uploaded image should be removed.
if cleanUp {
app.media.Delete(fName)
app.media.Delete(thumbPrefix + fName)
if thumbfName != "" {
app.media.Delete(thumbfName)
}
}
}()
// Create thumbnail from file for non-vector formats.
var (
thumbfName = fName
width = 0
height = 0
)
if !inArray(ext, vectorExts) {
isImage := inArray(ext, imageExts)
if isImage {
thumbFile, w, h, err := processImage(file)
if err != nil {
cleanUp = true
@ -100,7 +98,7 @@ func handleUploadMedia(c echo.Context) error {
height = h
// Upload thumbnail.
tf, err := app.media.Put(thumbPrefix+fName, typ, thumbFile)
tf, err := app.media.Put(thumbPrefix+fName, contentType, thumbFile)
if err != nil {
cleanUp = true
app.log.Printf("error saving thumbnail: %v", err)
@ -109,13 +107,19 @@ func handleUploadMedia(c echo.Context) error {
}
thumbfName = tf
}
if inArray(ext, vectorExts) {
thumbfName = fName
}
// Write to the DB.
meta := models.JSON{
meta := models.JSON{}
if isImage {
meta = models.JSON{
"width": width,
"height": height,
}
m, err := app.core.InsertMedia(fName, thumbfName, meta, app.constants.MediaProvider, app.media)
}
m, err := app.core.InsertMedia(fName, thumbfName, contentType, meta, app.constants.MediaUpload.Provider, app.media)
if err != nil {
cleanUp = true
return err
@ -139,7 +143,7 @@ func handleGetMedia(c echo.Context) error {
return c.JSON(http.StatusOK, okResp{out})
}
out, err := app.core.GetAllMedia(app.constants.MediaProvider, app.media)
out, err := app.core.GetAllMedia(app.constants.MediaUpload.Provider, app.media)
if err != nil {
return err
}

View file

@ -576,7 +576,7 @@ func handleSelfExportSubscriberData(c echo.Context) error {
{
Name: fname,
Content: b,
Header: manager.MakeAttachmentHeader(fname, "base64"),
Header: manager.MakeAttachmentHeader(fname, "base64", "application/json"),
},
},
}); err != nil {

View file

@ -162,6 +162,10 @@ func handleUpdateSettings(c echo.Context) error {
set.SecurityCaptchaSecret = cur.SecurityCaptchaSecret
}
for n, v := range set.UploadExtensions {
set.UploadExtensions[n] = strings.ToLower(strings.TrimPrefix(strings.TrimSpace(v), "."))
}
// Domain blocklist.
doms := make([]string, 0)
for _, d := range set.DomainBlocklist {

View file

@ -57,10 +57,11 @@ func handleSendTxMessage(c echo.Context) error {
m.Attachments = append(m.Attachments, models.Attachment{
Name: f.Filename,
Header: manager.MakeAttachmentHeader(f.Filename, "base64"),
Header: manager.MakeAttachmentHeader(f.Filename, "base64", f.Header.Get("Content-Type")),
Content: b,
})
}
} else if err := c.Bind(&m); err != nil {
return err
}

View file

@ -12,8 +12,41 @@ describe('Campaigns', () => {
cy.get('tbody td[data-label=Status]').should('have.length', 1);
});
it('Creates campaign', () => {
cy.get('a[data-cy=btn-new]').click();
// Fill fields.
cy.get('input[name=name]').clear().type('new-attach');
cy.get('input[name=subject]').clear().type('new-subject');
cy.get('input[name=from_email]').clear().type('new <from@email>');
cy.get('.list-selector input').click();
cy.get('.list-selector .autocomplete a').eq(0).click();
cy.get('button[data-cy=btn-continue]').click();
cy.wait(500);
cy.get('a[data-cy=btn-attach]').click();
cy.get('input[type=file]').attachFile('example.json');
cy.get('.modal button.is-primary').click();
cy.get('.modal .thumb a.link').click();
cy.get('button[data-cy=btn-save]').click();
cy.wait(500);
// Re-open and check that the file still exists.
cy.loginAndVisit('/campaigns');
cy.get('td[data-label=Status] a').eq(0).click();
cy.get('.b-tabs nav a').eq(1).click();
cy.get('div.field[data-cy=media]').contains('example');
// Start.
cy.get('button[data-cy=btn-start]').click();
cy.get('.modal button.is-primary').click();
cy.wait(500);
cy.get('tbody tr').eq(0).get('td[data-label=Status] .tag.running');
});
it('Edits campaign', () => {
cy.get('td[data-label=Status] a').click();
cy.get('td[data-label=Status] a').eq(1).click();
// Fill fields.
cy.get('input[name=name]').clear().type('new-name');
@ -196,7 +229,7 @@ describe('Campaigns', () => {
cy.wait(250);
// Verify the changes.
(function(n) {
(function (n) {
cy.location('pathname').then((p) => {
cy.request(`${apiUrl}/api/campaigns/${p.split('/').at(-1)}`).should((response) => {
const { data } = response.body;

View file

@ -725,6 +725,8 @@ section.campaigns {
}
}
.campaign
section.analytics {
.charts {
position: relative;
@ -775,18 +777,41 @@ section.analytics {
max-height: 90px;
overflow: hidden;
position: relative;
width: 250px;
min-height: 250px;
text-align: center;
.link {
display: block;
}
.ext {
display: block;
margin-top: 60px;
font-size: 2rem;
text-transform: uppercase;
}
.filename {
display: inline-block;
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
color: $grey;
}
img {
max-width: 120px;
max-width: 250px;
}
.caption {
background-color: rgba($white, .70);
color: $grey;
position: absolute;
bottom: 0;
left: 0;
right: 0;
padding: 2px 5px;
padding: 5px 15px;
white-space: nowrap;
overflow: hidden;
@ -818,6 +843,19 @@ section.analytics {
}
}
.modal .media-files {
.thumb {
min-width: 175px;
width: 175px;
min-height: 175px;
}
.gallery {
padding-left: 0;
padding-right: 0;
}
}
/* Template form */
.templates {
td .tag {

View file

@ -102,7 +102,7 @@
<b-modal scroll="keep" :aria-modal="true" :active.sync="isMediaVisible" :width="900">
<div class="modal-card content" style="width: auto">
<section expanded class="modal-card-body">
<media isModal @selected="onMediaSelect" />
<media is-modal @selected="onMediaSelect" />
</section>
</div>
</b-modal>

View file

@ -130,7 +130,7 @@
<div>
<p class="has-text-right">
<a href="#" @click.prevent="showHeaders" data-cy="btn-headers">
<a href="#" @click.prevent="onShowHeaders" data-cy="btn-headers">
<b-icon icon="plus" />{{ $t('settings.smtp.setCustomHeaders') }}
</a>
</p>
@ -183,17 +183,36 @@
:disabled="!canEdit"
/>
<div v-if="canEdit && form.content.contentType !== 'plain'" class="alt-body">
<p class="is-size-6 has-text-grey has-text-right">
<a v-if="form.altbody === null" href="#" @click.prevent="addAltBody">
<div class="columns">
<div class="column is-6">
<p v-if="!isAttachFieldVisible" class="is-size-6 has-text-grey">
<a href="#" @click.prevent="onShowAttachField()" data-cy="btn-attach">
<b-icon icon="file-upload-outline" size="is-small" />
{{ $t('campaigns.addAttachments') }}
</a>
</p>
<b-field v-if="isAttachFieldVisible" :label="$t('campaigns.attachments')"
label-position="on-border" expanded data-cy="media">
<b-taginput v-model="form.media" name="media" ellipsis icon="tag-outline"
ref="media" field="filename" @focus="onOpenAttach" :disabled="!canEdit" />
</b-field>
</div>
<div class="column has-text-right">
<p v-if="canEdit && form.content.contentType !== 'plain'"
class="is-size-6 has-text-grey">
<a v-if="form.altbody === null" href="#" @click.prevent="onAddAltBody">
<b-icon icon="text" size="is-small" /> {{ $t('campaigns.addAltText') }}
</a>
<a v-else href="#" @click.prevent="$utils.confirm(null, removeAltBody)">
<a v-else href="#" @click.prevent="$utils.confirm(null, onRemoveAltBody)">
<b-icon icon="trash-can-outline" size="is-small" />
{{ $t('campaigns.removeAltText') }}
</a>
</p>
<br />
</div>
</div>
<div v-if="canEdit && form.content.contentType !== 'plain'" class="alt-body">
<b-input v-if="form.altbody !== null" v-model="form.altbody"
type="textarea" :disabled="!canEdit" />
</div>
@ -251,6 +270,14 @@
</section>
</b-tab-item><!-- archive -->
</b-tabs>
<b-modal scroll="keep" :aria-modal="true" :active.sync="isAttachModalOpen" :width="900">
<div class="modal-card content" style="width: auto">
<section expanded class="modal-card-body">
<media is-modal @selected="onAttachSelect" />
</section>
</div>
</b-modal>
</section>
</template>
@ -262,6 +289,7 @@ import htmlToPlainText from 'textversionjs';
import ListSelector from '../components/ListSelector.vue';
import Editor from '../components/Editor.vue';
import Media from './Media.vue';
const TABS = ['campaign', 'content', 'archive'];
@ -269,6 +297,7 @@ export default Vue.extend({
components: {
ListSelector,
Editor,
Media,
},
data() {
@ -276,6 +305,8 @@ export default Vue.extend({
isNew: false,
isEditing: false,
isHeadersVisible: false,
isAttachFieldVisible: false,
isAttachModalOpen: false,
activeTab: 0,
data: {},
@ -297,6 +328,7 @@ export default Vue.extend({
sendAt: null,
content: { contentType: 'richtext', body: '' },
altbody: null,
media: [],
// Parsed Date() version of send_at from the API.
sendAtDate: null,
@ -314,18 +346,37 @@ export default Vue.extend({
return dayjs(s).format('YYYY-MM-DD HH:mm');
},
addAltBody() {
onAddAltBody() {
this.form.altbody = htmlToPlainText(this.form.content.body);
},
removeAltBody() {
onRemoveAltBody() {
this.form.altbody = null;
},
showHeaders() {
onShowHeaders() {
this.isHeadersVisible = !this.isHeadersVisible;
},
onShowAttachField() {
this.isAttachFieldVisible = true;
this.$nextTick(() => {
this.$refs.media.focus();
});
},
onOpenAttach() {
this.isAttachModalOpen = true;
},
onAttachSelect(o) {
if (this.form.media.some((m) => m.id === o.id)) {
return;
}
this.form.media.push(o);
},
isUnsaved() {
return this.data.body !== this.form.content.body
|| this.data.contentType !== this.form.content.contentType;
@ -395,6 +446,14 @@ export default Vue.extend({
// The structure that is populated by editor input event.
content: { contentType: data.contentType, body: data.body },
};
this.isAttachFieldVisible = this.form.media.length > 0;
this.form.media = this.form.media.map((f) => {
if (!f.id) {
return { ...f, filename: `${f.filename}` };
}
return f;
});
if (data.sendAt !== null) {
this.form.sendLater = true;
@ -419,6 +478,7 @@ export default Vue.extend({
body: this.form.content.body,
altbody: this.form.content.contentType !== 'plain' ? this.form.altbody : null,
subscribers: this.form.testEmails,
media: this.form.media.map((m) => m.id),
};
this.$api.testCampaign(data).then(() => {
@ -441,6 +501,7 @@ export default Vue.extend({
send_at: this.form.sendLater ? this.form.sendAtDate : null,
headers: this.form.headers,
template_id: this.form.templateId,
media: this.form.media.map((m) => m.id),
// body: this.form.body,
};
@ -469,6 +530,7 @@ export default Vue.extend({
archive: this.form.archive,
archive_template_id: this.form.archiveTemplateId,
archive_meta: this.form.archiveMeta,
media: this.form.media.map((m) => m.id),
};
let typMsg = 'globals.messages.updated';

View file

@ -426,6 +426,7 @@ export default Vue.extend({
archive: c.archive,
archive_template_id: c.archiveTemplateId,
archive_meta: c.archiveMeta,
media: c.media.map((m) => m.id),
};
this.$api.createCampaign(data).then((d) => {

View file

@ -16,7 +16,7 @@
v-model="form.files"
drag-drop
multiple
accept=".png,.jpg,.jpeg,.gif,.svg"
xaccept=".png,.jpg,.jpeg,.gif,.svg"
expanded>
<div class="has-text-centered section">
<p>
@ -47,10 +47,15 @@
<div class="thumbs">
<div v-for="m in group.items" :key="m.id" class="box thumb">
<a @click="(e) => onMediaSelect(m, e)" :href="m.url" target="_blank">
<img :src="m.thumbUrl" :title="m.filename" />
<a @click="(e) => onMediaSelect(m, e)" :href="m.url" target="_blank" class="link">
<img v-if="m.thumbUrl" :src="m.thumbUrl" :title="m.filename" />
<template v-else>
<span class="ext" :title="m.filename">{{ m.filename.split(".").pop() }}</span><br />
</template>
<span class="caption is-size-5" :title="m.filename">
{{ m.filename }}
</span>
</a>
<span class="caption is-size-7" :title="m.filename">{{ m.filename }}</span>
<div class="actions has-text-right">
<a :href="m.url" target="_blank">
@ -65,7 +70,6 @@
<hr />
</div>
</section>
</section>
</template>
@ -79,6 +83,7 @@ export default Vue.extend({
props: {
isModal: Boolean,
type: String,
},
data() {
@ -161,6 +166,10 @@ export default Vue.extend({
let lastStamp = '';
let lastIndex = 0;
this.media.forEach((m) => {
if (this.$props.type === 'image' && !m.thumbUrl) {
return;
}
const stamp = dayjs(m.createdAt).format('MMM YYYY');
if (stamp !== lastStamp) {
out.push({ title: stamp, items: [] });

View file

@ -1,11 +1,23 @@
<template>
<div class="items">
<div class="columns">
<div class="column">
<b-field :label="$t('settings.media.provider')" label-position="on-border">
<b-select v-model="data['upload.provider']" name="upload.provider">
<option value="filesystem">filesystem</option>
<option value="s3">s3</option>
</b-select>
</b-field>
</div>
<div class="column is-10">
<b-field :label="$t('settings.media.upload.extensions')" label-position="on-border"
expanded>
<b-taginput v-model="data['upload.extensions']" name="tags" ellipsis
icon="tag-outline" placeholder="jpg, png, gif .."></b-taginput>
</b-field>
</div>
</div>
<hr />
<div class="block" v-if="data['upload.provider'] === 'filesystem'">
<b-field :label="$t('settings.media.upload.path')" label-position="on-border"
@ -126,6 +138,7 @@ export default Vue.extend({
return {
data: this.form,
regDuration,
extensions: [],
};
},

1
go.mod
View file

@ -9,6 +9,7 @@ require (
github.com/gofrs/uuid v4.0.0+incompatible
github.com/google/uuid v1.3.0 // indirect
github.com/gorilla/feeds v1.1.1
github.com/h2non/filetype v1.1.3 // indirect
github.com/huandu/xstrings v1.4.0 // indirect
github.com/imdario/mergo v0.3.14 // indirect
github.com/jmoiron/sqlx v1.3.5

2
go.sum
View file

@ -34,6 +34,8 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gorilla/feeds v1.1.1 h1:HwKXxqzcRNg9to+BbvJog4+f3s/xzvtZXICcQGutYfY=
github.com/gorilla/feeds v1.1.1/go.mod h1:Nk0jZrvPFZX1OBe5NPiddPw7CfwF6Q9eqzaBbaightA=
github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU=
github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=

View file

@ -17,6 +17,8 @@
"bounces.unknownService": "Unknown service.",
"bounces.view": "View bounces",
"campaigns.addAltText": "Add alternate plain text message",
"campaigns.addAttachments": "Add attachments",
"campaigns.attachments": "Attachments",
"campaigns.archive": "Archive",
"campaigns.archiveEnable": "Publish to public archive",
"campaigns.archiveHelp": "Publish (running, paused, finished) the campaign message on the public archive.",
@ -435,6 +437,8 @@
"settings.media.s3.urlHelp": "Only change if using a custom S3 compatible backend like Minio.",
"settings.media.title": "Media uploads",
"settings.media.upload.path": "Upload path",
"settings.media.upload.extensions": "Permitted file extensions",
"settings.media.upload.extensionsHelp": "Add * to allow all extensions",
"settings.media.upload.pathHelp": "Path to the directory where media will be uploaded.",
"settings.media.upload.uri": "Upload URI",
"settings.media.upload.uriHelp": "Upload URI that is visible to the outside world. The media uploaded to upload_path will be publicly accessible under {root_url}, for instance, https://listmonk.yoursite.com/uploads.",

View file

@ -155,7 +155,7 @@ func (c *Core) GetArchivedCampaigns(offset, limit int) (models.Campaigns, int, e
}
// CreateCampaign creates a new campaign.
func (c *Core) CreateCampaign(o models.Campaign, listIDs []int) (models.Campaign, error) {
func (c *Core) CreateCampaign(o models.Campaign, listIDs []int, mediaIDs []int) (models.Campaign, error) {
uu, err := uuid.NewV4()
if err != nil {
c.log.Printf("error generating UUID: %v", err)
@ -183,6 +183,7 @@ func (c *Core) CreateCampaign(o models.Campaign, listIDs []int) (models.Campaign
o.Archive,
o.ArchiveTemplateID,
o.ArchiveMeta,
pq.Array(mediaIDs),
); err != nil {
if err == sql.ErrNoRows {
return models.Campaign{}, echo.NewHTTPError(http.StatusBadRequest, c.i18n.T("campaigns.noSubs"))
@ -202,7 +203,7 @@ func (c *Core) CreateCampaign(o models.Campaign, listIDs []int) (models.Campaign
}
// UpdateCampaign updates a campaign.
func (c *Core) UpdateCampaign(id int, o models.Campaign, listIDs []int, sendLater bool) (models.Campaign, error) {
func (c *Core) UpdateCampaign(id int, o models.Campaign, listIDs []int, mediaIDs []int, sendLater bool) (models.Campaign, error) {
_, err := c.q.UpdateCampaign.Exec(id,
o.Name,
o.Subject,
@ -219,7 +220,8 @@ func (c *Core) UpdateCampaign(id int, o models.Campaign, listIDs []int, sendLate
pq.Array(listIDs),
o.Archive,
o.ArchiveTemplateID,
o.ArchiveMeta)
o.ArchiveMeta,
pq.Array(mediaIDs))
if err != nil {
c.log.Printf("error updating campaign: %v", err)
return models.Campaign{}, echo.NewHTTPError(http.StatusInternalServerError,

View file

@ -7,6 +7,7 @@ import (
"github.com/knadh/listmonk/internal/media"
"github.com/knadh/listmonk/models"
"github.com/labstack/echo/v4"
"gopkg.in/volatiletech/null.v6"
)
// GetAllMedia returns all uploaded media.
@ -20,7 +21,10 @@ func (c *Core) GetAllMedia(provider string, s media.Store) ([]media.Media, error
for i := 0; i < len(out); i++ {
out[i].URL = s.GetURL(out[i].Filename)
out[i].ThumbURL = s.GetURL(out[i].Thumb)
if out[i].Thumb != "" {
out[i].ThumbURL = null.String{Valid: true, String: s.GetURL(out[i].Thumb)}
}
}
return out, nil
@ -40,13 +44,15 @@ func (c *Core) GetMedia(id int, uuid string, s media.Store) (media.Media, error)
}
out.URL = s.GetURL(out.Filename)
out.ThumbURL = s.GetURL(out.Thumb)
if out.Thumb != "" {
out.ThumbURL = null.String{Valid: true, String: s.GetURL(out.Thumb)}
}
return out, nil
}
// InsertMedia inserts a new media file into the DB.
func (c *Core) InsertMedia(fileName, thumbName string, meta models.JSON, provider string, s media.Store) (media.Media, error) {
func (c *Core) InsertMedia(fileName, thumbName, contentType string, meta models.JSON, provider string, s media.Store) (media.Media, error) {
uu, err := uuid.NewV4()
if err != nil {
c.log.Printf("error generating UUID: %v", err)
@ -56,7 +62,7 @@ func (c *Core) InsertMedia(fileName, thumbName string, meta models.JSON, provide
// Write to the DB.
var newID int
if err := c.q.InsertMedia.Get(&newID, uu, fileName, thumbName, provider, meta); err != nil {
if err := c.q.InsertMedia.Get(&newID, uu, fileName, thumbName, contentType, provider, meta); err != nil {
c.log.Printf("error inserting uploaded file to db: %v", err)
return media.Media{}, echo.NewHTTPError(http.StatusInternalServerError,
c.i18n.Ts("globals.messages.errorCreating", "name", "{globals.terms.media}", "error", pqErrMsg(err)))

View file

@ -33,6 +33,7 @@ type Store interface {
NextCampaigns(excludeIDs []int64) ([]*models.Campaign, error)
NextSubscribers(campID, limit int) ([]models.Subscriber, error)
GetCampaign(campID int) (*models.Campaign, error)
GetAttachment(mediaID int) (models.Attachment, error)
UpdateCampaignStatus(campID int, status string) error
CreateLink(url string) (string, error)
BlocklistSubscriber(id int64) error
@ -232,6 +233,11 @@ func (m *Manager) PushCampaignMessage(msg CampaignMessage) error {
t := time.NewTicker(pushTimeout)
defer t.Stop()
// Load any media/attachments.
if err := m.attachMedia(msg.Campaign); err != nil {
return err
}
select {
case m.campMsgQueue <- msg:
case <-t.C:
@ -364,6 +370,7 @@ func (m *Manager) worker() {
AltBody: msg.altBody,
Subscriber: msg.Subscriber,
Campaign: msg.Campaign,
Attachments: msg.Campaign.Attachments,
}
h := textproto.MIMEHeader{}
@ -549,6 +556,11 @@ func (m *Manager) addCampaign(c *models.Campaign) error {
return err
}
// Load any media/attachments.
if err := m.attachMedia(c); err != nil {
return err
}
// Add the campaign to the active map.
m.campsMut.Lock()
m.camps[c.ID] = c
@ -802,16 +814,34 @@ func (m *Manager) makeGnericFuncMap() template.FuncMap {
return f
}
func (m *Manager) attachMedia(c *models.Campaign) error {
// Load any media/attachments.
for _, mid := range []int64(c.MediaIDs) {
a, err := m.store.GetAttachment(int(mid))
if err != nil {
return fmt.Errorf("error fetching attachment %d on campaign %s: %v", mid, c.Name, err)
}
c.Attachments = append(c.Attachments, a)
}
return nil
}
// MakeAttachmentHeader is a helper function that returns a
// textproto.MIMEHeader tailored for attachments, primarily
// email. If no encoding is given, base64 is assumed.
func MakeAttachmentHeader(filename, encoding string) textproto.MIMEHeader {
func MakeAttachmentHeader(filename, encoding, contentType string) textproto.MIMEHeader {
if encoding == "" {
encoding = "base64"
}
if contentType == "" {
contentType = "application/octet-stream"
}
h := textproto.MIMEHeader{}
h.Set("Content-Disposition", "attachment; filename="+filename)
h.Set("Content-Type", "application/json; name=\""+filename+"\"")
h.Set("Content-Type", fmt.Sprintf("%s; name=\""+filename+"\"", contentType))
h.Set("Content-Transfer-Encoding", encoding)
return h
}

View file

@ -12,9 +12,10 @@ type Media struct {
ID int `db:"id" json:"id"`
UUID string `db:"uuid" json:"uuid"`
Filename string `db:"filename" json:"filename"`
Thumb string `db:"thumb" json:"thumb"`
ContentType string `db:"content_type" json:"content_type"`
Thumb string `db:"thumb" json:"-"`
CreatedAt null.Time `db:"created_at" json:"created_at"`
ThumbURL string `json:"thumb_url"`
ThumbURL null.String `json:"thumb_url"`
Provider string `json:"provider"`
Meta models.JSON `db:"meta" json:"meta"`
URL string `json:"url"`

View file

@ -11,6 +11,7 @@ func V2_5_0(db *sqlx.DB, fs stuffbin.FileSystem, ko *koanf.Koanf) error {
// Insert new preference settings.
if _, err := db.Exec(`
INSERT INTO settings (key, value) VALUES
('upload.extensions', '["jpg","jpeg","png","gif","svg","*"]'),
('app.enable_public_archive_rss_content', 'false'),
('bounce.actions', '{"soft": {"count": 2, "action": "none"}, "hard": {"count": 2, "action": "blocklist"}, "complaint" : {"count": 2, "action": "blocklist"}}')
ON CONFLICT DO NOTHING;
@ -18,7 +19,35 @@ func V2_5_0(db *sqlx.DB, fs stuffbin.FileSystem, ko *koanf.Koanf) error {
return err
}
if _, err := db.Exec(`DELETE FROM settings WHERE key IN ('bounce.count', 'bounce.action');`); err != nil {
if _, err := db.Exec(`
DELETE FROM settings WHERE key IN ('bounce.count', 'bounce.action');
-- Add the content_type column.
ALTER TABLE media ADD COLUMN IF NOT EXISTS content_type TEXT NOT NULL DEFAULT 'application/octet-stream';
-- Fill the content type column for existing files (which would only be images at this point).
UPDATE media SET content_type = CASE
WHEN LOWER(SUBSTRING(filename FROM '.([^.]+)$')) = 'svg' THEN 'image/svg+xml'
ELSE 'image/' || LOWER(SUBSTRING(filename FROM '.([^.]+)$'))
END;
`); err != nil {
return err
}
if _, err := db.Exec(`
CREATE TABLE IF NOT EXISTS campaign_media (
campaign_id INTEGER REFERENCES campaigns(id) ON DELETE CASCADE ON UPDATE CASCADE,
-- Media items may be deleted, so media_id is nullable
-- and a copy of the original name is maintained here.
media_id INTEGER NULL REFERENCES media(id) ON DELETE SET NULL ON UPDATE CASCADE,
filename TEXT NOT NULL DEFAULT ''
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_camp_media_id ON campaign_media (campaign_id, media_id);
CREATE INDEX IF NOT EXISTS idx_camp_media_camp_id ON campaign_media(campaign_id);
`); err != nil {
return err
}

View file

@ -261,6 +261,12 @@ type Campaign struct {
SubjectTpl *txttpl.Template `json:"-"`
AltBodyTpl *template.Template `json:"-"`
// List of media (attachment) IDs obtained from the next-campaign query
// while sending a campaign.
MediaIDs pq.Int64Array `json:"-" db:"media_id"`
// Fetched bodies of the attachments.
Attachments []Attachment `json:"-" db:"-"`
// Pseudofield for getting the total number of subscribers
// in searches and queries.
Total int `db:"total" json:"-"`
@ -279,6 +285,7 @@ type CampaignMeta struct {
// campaign-list associations with a historical record of id + name that persist
// even after a list is deleted.
Lists types.JSONText `db:"lists" json:"lists"`
Media types.JSONText `db:"media" json:"media"`
StartedAt null.Time `db:"started_at" json:"started_at"`
ToSend int `db:"to_send" json:"to_send"`
@ -506,6 +513,7 @@ func (camps Campaigns) LoadStats(stmt *sqlx.Stmt) error {
camps[i].Views = c.Views
camps[i].Clicks = c.Clicks
camps[i].Bounces = c.Bounces
camps[i].Media = c.Media
}
}

View file

@ -38,6 +38,7 @@ type Settings struct {
SecurityCaptchaSecret string `json:"security.captcha_secret"`
UploadProvider string `json:"upload.provider"`
UploadExtensions []string `json:"upload.extensions"`
UploadFilesystemUploadPath string `json:"upload.filesystem.upload_path"`
UploadFilesystemUploadURI string `json:"upload.filesystem.upload_uri"`
UploadS3URL string `json:"upload.s3.url"`

View file

@ -475,8 +475,15 @@ counts AS (
),
camp AS (
INSERT INTO campaigns (uuid, type, name, subject, from_email, body, altbody, content_type, send_at, headers, tags, messenger, template_id, to_send, max_subscriber_id, archive, archive_template_id, archive_meta)
SELECT $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, (SELECT id FROM tpl), (SELECT to_send FROM counts), (SELECT max_sub_id FROM counts), $15, (CASE WHEN $16 = 0 THEN (SELECT id FROM tpl) ELSE $16 END), $17
SELECT $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12,
(SELECT id FROM tpl), (SELECT to_send FROM counts),
(SELECT max_sub_id FROM counts), $15,
(CASE WHEN $16 = 0 THEN (SELECT id FROM tpl) ELSE $16 END), $17
RETURNING id
),
med AS (
INSERT INTO campaign_media (campaign_id, media_id, filename)
(SELECT (SELECT id FROM camp), id, filename FROM media WHERE id=ANY($18::INT[]))
)
INSERT INTO campaign_lists (campaign_id, list_id, list_name)
(SELECT (SELECT id FROM camp), id, name FROM lists WHERE id=ANY($14::INT[]))
@ -492,7 +499,8 @@ INSERT INTO campaign_lists (campaign_id, list_id, list_name)
SELECT c.id, c.uuid, c.name, c.subject, c.from_email,
c.messenger, c.started_at, c.to_send, c.sent, c.type,
c.body, c.altbody, c.send_at, c.headers, c.status, c.content_type, c.tags,
c.template_id, c.archive, c.archive_template_id, c.archive_meta, c.created_at, c.updated_at,
c.template_id, c.archive, c.archive_template_id, c.archive_meta,
c.created_at, c.updated_at,
COUNT(*) OVER () AS total,
(
SELECT COALESCE(ARRAY_TO_JSON(ARRAY_AGG(l)), '[]') FROM (
@ -536,7 +544,12 @@ SELECT COUNT(*) OVER () AS total, campaigns.*,
WITH lists AS (
SELECT campaign_id, JSON_AGG(JSON_BUILD_OBJECT('id', list_id, 'name', list_name)) AS lists FROM campaign_lists
WHERE campaign_id = ANY($1) GROUP BY campaign_id
), views AS (
),
media AS (
SELECT campaign_id, JSON_AGG(JSON_BUILD_OBJECT('id', media_id, 'filename', filename)) AS media FROM campaign_media
WHERE campaign_id = ANY($1) GROUP BY campaign_id
),
views AS (
SELECT campaign_id, COUNT(campaign_id) as num FROM campaign_views
WHERE campaign_id = ANY($1)
GROUP BY campaign_id
@ -555,9 +568,11 @@ SELECT id as campaign_id,
COALESCE(v.num, 0) AS views,
COALESCE(c.num, 0) AS clicks,
COALESCE(b.num, 0) AS bounces,
COALESCE(l.lists, '[]') AS lists
COALESCE(l.lists, '[]') AS lists,
COALESCE(m.media, '[]') AS media
FROM (SELECT id FROM UNNEST($1) AS id) x
LEFT JOIN lists AS l ON (l.campaign_id = id)
LEFT JOIN media AS m ON (m.campaign_id = id)
LEFT JOIN views AS v ON (v.campaign_id = id)
LEFT JOIN clicks AS c ON (c.campaign_id = id)
LEFT JOIN bounces AS b ON (b.campaign_id = id)
@ -602,6 +617,12 @@ campLists AS (
INNER JOIN campaign_lists ON (campaign_lists.list_id = lists.id)
WHERE campaign_lists.campaign_id = ANY(SELECT id FROM camps)
),
campMedia AS (
-- Get the list_ids and their optin statuses for the campaigns found in the previous step.
SELECT campaign_id, ARRAY_AGG(campaign_media.media_id)::INT[] AS media_id FROM campaign_media
WHERE campaign_id = ANY(SELECT id FROM camps) AND media_id IS NOT NULL
GROUP BY campaign_id
),
counts AS (
-- For each campaign above, get the total number of subscribers and the max_subscriber_id
-- across all its lists.
@ -636,7 +657,7 @@ u AS (
FROM (SELECT * FROM counts) co
WHERE ca.id = co.campaign_id
)
SELECT * FROM camps;
SELECT camps.*, campMedia.media_id FROM camps LEFT JOIN campMedia ON (campMedia.campaign_id = camps.id);
-- name: get-campaign-analytics-unique-counts
WITH intval AS (
@ -769,9 +790,18 @@ WITH camp AS (
updated_at=NOW()
WHERE id = $1 RETURNING id
),
d AS (
clists AS (
-- Reset list relationships
DELETE FROM campaign_lists WHERE campaign_id = $1 AND NOT(list_id = ANY($14))
),
med AS (
DELETE FROM campaign_media WHERE campaign_id = $1
AND media_id IS NULL or NOT(media_id = ANY($18)) RETURNING media_id
),
medi AS (
INSERT INTO campaign_media (campaign_id, media_id, filename)
(SELECT $1 AS campaign_id, id, filename FROM media WHERE id=ANY($18::INT[]))
ON CONFLICT (campaign_id, media_id) DO NOTHING
)
INSERT INTO campaign_lists (campaign_id, list_id, list_name)
(SELECT $1 as campaign_id, id, name FROM lists WHERE id=ANY($14::INT[]))
@ -872,7 +902,7 @@ SELECT id FROM tpl;
-- media
-- name: insert-media
INSERT INTO media (uuid, filename, thumb, provider, meta, created_at) VALUES($1, $2, $3, $4, $5, NOW()) RETURNING id;
INSERT INTO media (uuid, filename, thumb, content_type, provider, meta, created_at) VALUES($1, $2, $3, $4, $5, $6, NOW()) RETURNING id;
-- name: get-all-media
SELECT * FROM media WHERE provider=$1 ORDER BY created_at DESC;

View file

@ -144,11 +144,26 @@ CREATE TABLE media (
uuid uuid NOT NULL UNIQUE,
provider TEXT NOT NULL DEFAULT '',
filename TEXT NOT NULL,
content_type TEXT NOT NULL DEFAULT 'application/octet-stream',
thumb TEXT NOT NULL,
meta JSONB NOT NULL DEFAULT '{}',
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
);
-- campaign_media
DROP TABLE IF EXISTS campaign_media CASCADE;
CREATE TABLE campaign_media (
campaign_id INTEGER REFERENCES campaigns(id) ON DELETE CASCADE ON UPDATE CASCADE,
-- Media items may be deleted, so media_id is nullable
-- and a copy of the original name is maintained here.
media_id INTEGER NULL REFERENCES media(id) ON DELETE SET NULL ON UPDATE CASCADE,
filename TEXT NOT NULL DEFAULT ''
);
DROP INDEX IF EXISTS idx_camp_media_id; CREATE UNIQUE INDEX idx_camp_media_id ON campaign_media (campaign_id, media_id);
DROP INDEX IF EXISTS idx_camp_media_camp_id; CREATE INDEX idx_camp_media_camp_id ON campaign_media(campaign_id);
-- links
DROP TABLE IF EXISTS links CASCADE;
CREATE TABLE links (
@ -213,6 +228,8 @@ INSERT INTO settings (key, value) VALUES
('security.captcha_key', '""'),
('security.captcha_secret', '""'),
('upload.provider', '"filesystem"'),
('upload.max_file_size', '5000'),
('upload.extensions', '["jpg","jpeg","png","gif","svg","*"]'),
('upload.filesystem.upload_path', '"uploads"'),
('upload.filesystem.upload_uri', '"/uploads"'),
('upload.s3.url', '"https://ap-south-1.s3.amazonaws.com"'),