beenull/listmonk
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Refactor and fix media uploads.

Browse source 
- Fix path related issues in filesystem and S3.
- Add checks for S3 "/" path prefix.
- Add support for custom S3 domain names.
- Remove obsolete `width` and `height` columns from media table (breaking)
- Add `provider` field to media table (breaking)
This commit is contained in:
Kailash Nadh 2020-07-05 17:35:05 +05:30
parent 7f9a811897
commit 24192a327f
15 changed files with 91 additions and 69 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
admin.go
View file

@ -11,9 +11,10 @@ import (
)
type configScript struct {
RootURL string `json:"rootURL"`
FromEmail string `json:"fromEmail"`
Messengers []string `json:"messengers"`
RootURL string `json:"rootURL"`
FromEmail string `json:"fromEmail"`
Messengers []string `json:"messengers"`
MediaProvider string `json:"media_provider"`
}
// handleGetConfigScript returns general configuration as a Javascript
@ -22,9 +23,10 @@ func handleGetConfigScript(c echo.Context) error {
var (
app = c.Get("app").(*App)
out = configScript{
RootURL: app.constants.RootURL,
FromEmail: app.constants.FromEmail,
Messengers: app.manager.GetMessengerNames(),
RootURL: app.constants.RootURL,
FromEmail: app.constants.FromEmail,
Messengers: app.manager.GetMessengerNames(),
MediaProvider: app.constants.MediaProvider,
}
b = bytes.Buffer{}

15
config.toml.sample
View file

@ -166,20 +166,23 @@ provider = "filesystem"
aws_secret_access_key = ""
# AWS Region where S3 bucket is hosted.
aws_default_region="ap-south-1"
aws_default_region = "ap-south-1"
# Bucket name.
bucket=""
bucket = ""
# Path where the files will be stored inside bucket. Empty for root.
bucket_path=""
# Path where the files will be stored inside bucket. Default is "/".
bucket_path = "/"
# Optional full URL to the bucket. eg: https://files.mycustom.com
bucket_url = ""
# "private" or "public".
bucket_type="public"
bucket_type = "public"
# (Optional) Specify TTL (in seconds) for the generated presigned URL.
# Expiry value is used only if the bucket is private.
expiry="86400"
expiry = 86400
[upload.filesystem]
# Path to the uploads directory where media will be uploaded.

2
frontend/README.md vendored
View file

@ -5,7 +5,7 @@ It's best if the `listmonk/frontend` directory is opened in an IDE as a separate
For developer setup instructions, refer to the main project's README.
## Globals
`main.js` is where Buefy is injected globally into Vue. In addition two controllers, `$api` (collection of API calls from `api/index.js`) and `$utils` (util functions from `util.js`), are also attached globaly to Vue. They are accessible within Vue as `this.$api` and `this.$utils`.
`main.js` is where Buefy is injected globally into Vue. In addition two controllers, `$api` (collection of API calls from `api/index.js`), `$utils` (util functions from `util.js`), `$serverConfig` (loaded form /api/config.js) are also attached globaly to Vue. They are accessible within Vue as `this.$api` and `this.$utils`.
Some constants are defined in `constants.js`.

3
frontend/src/api/index.js
View file

@ -22,8 +22,7 @@ const http = axios.create({
return resp;
}
const data = humps.camelizeKeys(resp.data);
return data;
return humps.camelizeKeys(resp.data);
},
],

2
frontend/src/components/Editor.vue
View file

@ -164,7 +164,7 @@ export default {
},
onMediaSelect(m) {
this.$refs.quill.quill.insertEmbed(10, 'image', m.uri);
this.$refs.quill.quill.insertEmbed(10, 'image', m.url);
},
},

4
frontend/src/main.js
View file

@ -1,5 +1,6 @@
import Vue from 'vue';
import Buefy from 'buefy';
import humps from 'humps';
import App from './App.vue';
import router from './router';
@ -14,6 +15,9 @@ Vue.config.productionTip = false;
Vue.prototype.$api = api;
Vue.prototype.$utils = utils;
// window.CONFIG is loaded from /api/config.js directly in a <script> tag.
Vue.prototype.$serverConfig = humps.camelizeKeys(window.CONFIG);
new Vue({
router,
store,

1
frontend/src/utils.js
View file

@ -91,6 +91,7 @@ export default class utils {
message: msg,
type: !typ ? 'is-success' : typ,
queue: false,
duration: 3000,
});
};
}

10
frontend/src/views/Media.vue
View file

@ -2,6 +2,8 @@
<section class="media-files">
<h1 class="title is-4">Media
<span v-if="media.length > 0">({{ media.length }})</span>
<span class="has-text-grey-light"> / {{ $serverConfig.mediaProvider }}</span>
</h1>
<b-loading :active="isProcessing || loading.media"></b-loading>
@ -45,16 +47,16 @@
<div class="thumbs">
<div v-for="m in group.items" :key="m.id" class="box thumb">
<a @click="(e) => onMediaSelect(m, e)" :href="m.uri" target="_blank">
<img :src="m.thumbUri" :title="m.filename" />
<a @click="(e) => onMediaSelect(m, e)" :href="m.url" target="_blank">
<img :src="m.thumbUrl" :title="m.filename" />
</a>
<span class="caption is-size-7" :title="m.filename">{{ m.filename }}</span>
<div class="actions has-text-right">
<a :href="m.uri" target="_blank">
<a :href="m.url" target="_blank">
<b-icon icon="arrow-top-right" size="is-small" />
</a>
<a href="#" @click.prevent="deleteMedia(m.id)">
<a href="#" @click.prevent="$utils.confirm(null, () => deleteMedia(m.id))">
<b-icon icon="trash-can-outline" size="is-small" />
</a>
</div>

5
init.go
View file

@ -146,6 +146,8 @@ type constants struct {
ViewTrackURL string
OptinURL string
MessageURL string
MediaProvider string
}
func initConstants() *constants {
@ -159,6 +161,7 @@ func initConstants() *constants {
}
c.RootURL = strings.TrimRight(c.RootURL, "/")
c.Privacy.Exportable = maps.StringSliceToLookupMap(ko.Strings("privacy.exportable"))
c.MediaProvider = ko.String("upload.provider")
// Static URLS.
// url.com/subscription/{campaign_uuid}/{subscriber_uuid}
@ -175,7 +178,6 @@ func initConstants() *constants {
// url.com/campaign/{campaign_uuid}/{subscriber_uuid}/px.png
c.ViewTrackURL = fmt.Sprintf("%s/campaign/%%s/%%s/px.png", c.RootURL)
return &c
}
@ -272,6 +274,7 @@ func initMediaStore() media.Store {
case "filesystem":
var opts filesystem.Opts
ko.Unmarshal("upload.filesystem", &opts)
opts.RootURL = ko.String("app.root")
opts.UploadPath = filepath.Clean(opts.UploadPath)
opts.UploadURI = filepath.Clean(opts.UploadURI)
uplder, err := filesystem.NewDiskStore(opts)

10
internal/media/media.go
View file

@ -11,14 +11,14 @@ type Media struct {
ID int `db:"id" json:"id"`
UUID string `db:"uuid" json:"uuid"`
Filename string `db:"filename" json:"filename"`
Width int `db:"width" json:"width"`
Height int `db:"height" json:"height"`
Thumb string `db:"thumb" json:"thumb"`
CreatedAt null.Time `db:"created_at" json:"created_at"`
ThumbURI string `json:"thumb_uri"`
URI string `json:"uri"`
ThumbURL string `json:"thumb_url"`
Provider string `json:"provider"`
URL string `json:"url"`
}
// Store represents set of methods to perform upload/delete operations.
// Store represents functions to store and retrieve media (files).
type Store interface {
Put(string, string, io.ReadSeeker) (string, error)
Delete(string) error

25
internal/media/providers/filesystem/filesystem.go
View file

@ -19,6 +19,7 @@ const tmpFilePrefix = "listmonk"
type Opts struct {
UploadPath string `koanf:"upload_path"`
UploadURI string `koanf:"upload_uri"`
RootURL string `koanf:"root_url"`
}
// Client implements `media.Store`
@ -26,6 +27,12 @@ type Client struct {
opts Opts
}
// This matches filenames, sans extensions, of the format
// filename_(number). The number is incremented in case
// new file uploads conflict with existing filenames
// on the filesystem.
var fnameRegexp = regexp.MustCompile(`(.+?)_([0-9]+)$`)
// NewDiskStore initialises store for Filesystem provider.
func NewDiskStore(opts Opts) (media.Store, error) {
return &Client{
@ -34,7 +41,7 @@ func NewDiskStore(opts Opts) (media.Store, error) {
}
// Put accepts the filename, the content type and file object itself and stores the file in disk.
func (e *Client) Put(filename string, cType string, src io.ReadSeeker) (string, error) {
func (c *Client) Put(filename string, cType string, src io.ReadSeeker) (string, error) {
var out *os.File
// There's no explicit name. Use the one posted in the HTTP request.
if filename == "" {
@ -44,7 +51,7 @@ func (e *Client) Put(filename string, cType string, src io.ReadSeeker) (string,
}
}
// Get the directory path
dir := getDir(e.opts.UploadPath)
dir := getDir(c.opts.UploadPath)
filename = assertUniqueFilename(dir, filename)
o, err := os.OpenFile(filepath.Join(dir, filename), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0664)
if err != nil {
@ -60,13 +67,13 @@ func (e *Client) Put(filename string, cType string, src io.ReadSeeker) (string,
}
// Get accepts a filename and retrieves the full path from disk.
func (e *Client) Get(name string) string {
return fmt.Sprintf("%s/%s", e.opts.UploadURI, name)
func (c *Client) Get(name string) string {
return fmt.Sprintf("%s%s/%s", c.opts.RootURL, c.opts.UploadURI, name)
}
// Delete accepts a filename and removes it from disk.
func (e *Client) Delete(file string) error {
dir := getDir(e.opts.UploadPath)
func (c *Client) Delete(file string) error {
dir := getDir(c.opts.UploadPath)
err := os.Remove(filepath.Join(dir, file))
if err != nil {
return err
@ -74,12 +81,6 @@ func (e *Client) Delete(file string) error {
return nil
}
// This matches filenames, sans extensions, of the format
// filename_(number). The number is incremented in case
// new file uploads conflict with existing filenames
// on the filesystem.
var fnameRegexp = regexp.MustCompile(`(.+?)_([0-9]+)$`)
// assertUniqueFilename takes a file path and check if it exists on the disk. If it doesn't,
// it returns the same name and if it does, it adds a small random hash to the filename
// and returns that.

54
internal/media/providers/s3/s3.go
View file

@ -4,13 +4,14 @@ import (
"errors"
"fmt"
"io"
"strings"
"time"
"github.com/knadh/listmonk/internal/media"
"github.com/rhnvrm/simples3"
)
const amznS3PublicURL = "https://%s.s3.%s.amazonaws.com/%s"
const amznS3PublicURL = "https://%s.s3.%s.amazonaws.com%s"
// Opts represents AWS S3 specific params
type Opts struct {
@ -19,6 +20,7 @@ type Opts struct {
Region string `koanf:"aws_default_region"`
Bucket string `koanf:"bucket"`
BucketPath string `koanf:"bucket_path"`
BucketURL string `koanf:"bucket_url"`
BucketType string `koanf:"bucket_type"`
Expiry int `koanf:"expiry"`
}
@ -54,55 +56,61 @@ func NewS3Store(opts Opts) (media.Store, error) {
}
// Put takes in the filename, the content type and file object itself and uploads to S3.
func (e *Client) Put(name string, cType string, file io.ReadSeeker) (string, error) {
func (c *Client) Put(name string, cType string, file io.ReadSeeker) (string, error) {
// Upload input parameters
upParams := simples3.UploadInput{
Bucket: e.opts.Bucket,
ObjectKey: getBucketPath(e.opts.BucketPath, name),
Bucket: c.opts.Bucket,
ContentType: cType,
FileName: name,
Body: file,
// Paths inside the bucket should not start with /.
ObjectKey: strings.TrimPrefix(makeBucketPath(c.opts.BucketPath, name), "/"),
}
// Perform an upload.
_, err := e.s3.FileUpload(upParams)
if err != nil {
if _, err := c.s3.FileUpload(upParams); err != nil {
return "", err
}
return name, nil
}
// Get accepts the filename of the object stored and retrieves from S3.
func (e *Client) Get(name string) string {
func (c *Client) Get(name string) string {
// Generate a private S3 pre-signed URL if it's a private bucket.
if e.opts.BucketType == "private" {
url := e.s3.GeneratePresignedURL(simples3.PresignedInput{
Bucket: e.opts.Bucket,
ObjectKey: getBucketPath(e.opts.BucketPath, name),
if c.opts.BucketType == "private" {
url := c.s3.GeneratePresignedURL(simples3.PresignedInput{
Bucket: c.opts.Bucket,
ObjectKey: makeBucketPath(c.opts.BucketPath, name),
Method: "GET",
Timestamp: time.Now(),
ExpirySeconds: e.opts.Expiry,
ExpirySeconds: c.opts.Expiry,
})
return url
}
// Generate a public S3 URL if it's a public bucket.
url := fmt.Sprintf(amznS3PublicURL, e.opts.Bucket, e.opts.Region, getBucketPath(e.opts.BucketPath, name))
url := ""
if c.opts.BucketURL != "" {
url = c.opts.BucketURL + makeBucketPath(c.opts.BucketPath, name)
} else {
url = fmt.Sprintf(amznS3PublicURL, c.opts.Bucket, c.opts.Region,
makeBucketPath(c.opts.BucketPath, name))
}
return url
}
// Delete accepts the filename of the object and deletes from S3.
func (e *Client) Delete(name string) error {
err := e.s3.FileDelete(simples3.DeleteInput{
Bucket: e.opts.Bucket,
ObjectKey: getBucketPath(e.opts.BucketPath, name),
func (c *Client) Delete(name string) error {
err := c.s3.FileDelete(simples3.DeleteInput{
Bucket: c.opts.Bucket,
ObjectKey: strings.TrimPrefix(makeBucketPath(c.opts.BucketPath, name), "/"),
})
return err
}
// getBucketPath constructs the key for the object stored in S3.
// If path is empty, the key is the combination of root of S3 bucket and filename.
func getBucketPath(path string, name string) string {
if path == "" {
return fmt.Sprintf("%s", name)
func makeBucketPath(bucketPath string, name string) string {
if bucketPath == "/" {
return "/" + name
}
return fmt.Sprintf("%s/%s", path, name)
return fmt.Sprintf("%s/%s", bucketPath, name)
}

8
media.go
View file

@ -99,7 +99,7 @@ func handleUploadMedia(c echo.Context) error {
}
// Write to the DB.
if _, err := app.queries.InsertMedia.Exec(uu, fName, thumbfName, 0, 0); err != nil {
if _, err := app.queries.InsertMedia.Exec(uu, fName, thumbfName, app.constants.MediaProvider); err != nil {
cleanUp = true
app.log.Printf("error inserting uploaded file to db: %v", err)
return echo.NewHTTPError(http.StatusInternalServerError,
@ -115,14 +115,14 @@ func handleGetMedia(c echo.Context) error {
out = []media.Media{}
)
if err := app.queries.GetMedia.Select(&out); err != nil {
if err := app.queries.GetMedia.Select(&out, app.constants.MediaProvider); err != nil {
return echo.NewHTTPError(http.StatusInternalServerError,
fmt.Sprintf("Error fetching media list: %s", pqErrMsg(err)))
}
for i := 0; i < len(out); i++ {
out[i].URI = app.media.Get(out[i].Filename)
out[i].ThumbURI = app.media.Get(thumbPrefix + out[i].Filename)
out[i].URL = app.media.Get(out[i].Filename)
out[i].ThumbURL = app.media.Get(out[i].Thumb)
}
return c.JSON(http.StatusOK, okResp{out})

4
queries.sql
View file

@ -654,10 +654,10 @@ UPDATE campaigns SET template_id = (SELECT id FROM def) WHERE (SELECT id FROM tp
-- media
-- name: insert-media
INSERT INTO media (uuid, filename, thumb, width, height, created_at) VALUES($1, $2, $3, $4, $5, NOW());
INSERT INTO media (uuid, filename, thumb, provider, created_at) VALUES($1, $2, $3, $4, NOW());
-- name: get-media
SELECT * FROM media ORDER BY created_at DESC;
SELECT * FROM media WHERE provider=$1 ORDER BY created_at DESC;
-- name: delete-media
DELETE FROM media WHERE id=$1 RETURNING filename;

3
schema.sql
View file

@ -128,10 +128,9 @@ DROP TABLE IF EXISTS media CASCADE;
CREATE TABLE media (
id SERIAL PRIMARY KEY,
uuid uuid NOT NULL UNIQUE,
provider TEXT NOT NULL,
filename TEXT NOT NULL,
thumb TEXT NOT NULL,
width INT NOT NULL DEFAULT 0,
height INT NOT NULL DEFAULT 0,
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
);