beenull/listmonk
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Add custom S3 backend support (eg: Minio) to media uploads

Browse source 
- Introduce a new S3 backend URL on the settings UI
- Add DB migration to populate S3 URL for existing S3 settings
- Refactor and fix URL formatting

Closes #139
This commit is contained in:
Kailash Nadh 2021-08-15 12:15:00 +05:30
parent 923b882f05
commit d6d1883587
20 changed files with 102 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
cmd/init.go
View file

@ -452,7 +452,7 @@ func initPostbackMessengers(m *manager.Manager) []messenger.Messenger {
func initMediaStore() media.Store {
switch provider := ko.String("upload.provider"); provider {
case "s3":
var o s3.Opts
var o s3.Opt
ko.Unmarshal("upload.s3", &o)
up, err := s3.NewS3Store(o)
if err != nil {

1
cmd/settings.go
View file

@ -42,6 +42,7 @@ type settings struct {
UploadProvider string `json:"upload.provider"`
UploadFilesystemUploadPath string `json:"upload.filesystem.upload_path"`
UploadFilesystemUploadURI string `json:"upload.filesystem.upload_uri"`
UploadS3URL string `json:"upload.s3.url"`
UploadS3AwsAccessKeyID string `json:"upload.s3.aws_access_key_id"`
UploadS3AwsDefaultRegion string `json:"upload.s3.aws_default_region"`
UploadS3AwsSecretAccessKey string `json:"upload.s3.aws_secret_access_key,omitempty"`

23
frontend/src/views/Settings.vue
View file

@ -18,7 +18,7 @@
<hr />
<section class="wrap-small">
<form @submit.prevent="onSubmit" v-if="!isLoading">
<form @submit.prevent="onSubmit">
<b-tabs type="is-boxed" :animated="false">
<b-tab-item :label="$t('settings.general.name')" label-position="on-border">
<div class="items">
@ -213,7 +213,7 @@
<div class="column is-3">
<b-field :label="$t('settings.media.s3.region')"
label-position="on-border" expanded>
<b-input v-model="form['upload.s3.aws_default_region']"
<b-input v-model="form['upload.s3.aws_default_region']" @input="onS3URLChange"
name="upload.s3.aws_default_region"
:maxlength="200" placeholder="ap-south-1" />
</b-field>
@ -254,7 +254,7 @@
<b-field grouped>
<b-field :label="$t('settings.media.s3.bucket')"
label-position="on-border" expanded>
<b-input v-model="form['upload.s3.bucket']"
<b-input v-model="form['upload.s3.bucket']" @input="onS3URLChange"
name="upload.s3.bucket" :maxlength="200" placeholder="" />
</b-field>
<b-field :label="$t('settings.media.s3.bucketPath')"
@ -276,6 +276,16 @@
placeholder="14d" :pattern="regDuration" :maxlength="10" />
</b-field>
</div>
<div class="column">
<b-field :label="$t('settings.media.s3.url')"
label-position="on-border"
:message="$t('settings.media.s3.urlHelp')" expanded>
<b-input v-model="form['upload.s3.url']"
name="upload.s3.url"
:disabled="!form['upload.s3.bucket']"
placeholder="https://s3.region.amazonaws.com" :maxlength="200" />
</b-field>
</div>
</div>
</div><!-- s3 -->
</div>
@ -786,6 +796,13 @@ export default Vue.extend({
this.form.messengers.splice(i, 1);
},
onS3URLChange() {
// If a custom non-AWS URL has been entered, don't update it automatically.
if (this.form['upload.s3.url'] !== '' && !this.form['upload.s3.url'].match(/amazonaws\.com/)) {
return;
}
this.form['upload.s3.url'] = `https://s3.${this.form['upload.s3.aws_default_region']}.amazonaws.com`;
},
onSubmit() {
const form = JSON.parse(JSON.stringify(this.form));

2
go.mod
View file

@ -19,7 +19,7 @@ require (
github.com/mailru/easyjson v0.7.6
github.com/mitchellh/copystructure v1.1.2 // indirect
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
github.com/rhnvrm/simples3 v0.5.0
github.com/rhnvrm/simples3 v0.7.0
github.com/spf13/pflag v1.0.5
github.com/yuin/goldmark v1.3.4
golang.org/x/mod v0.3.0

2
go.sum
View file

@ -84,6 +84,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rhnvrm/simples3 v0.5.0 h1:X+WX0hqoKScdoJAw/G3GArfZ6Ygsn8q+6MdocTMKXOw=
github.com/rhnvrm/simples3 v0.5.0/go.mod h1:Y+3vYm2V7Y4VijFoJHHTrja6OgPrJ2cBti8dPGkC3sA=
github.com/rhnvrm/simples3 v0.7.0 h1:KSEuKw0eGC5vltLW8ChLvjko+aUr0HbGet+bZHdwfMo=
github.com/rhnvrm/simples3 v0.7.0/go.mod h1:Y+3vYm2V7Y4VijFoJHHTrja6OgPrJ2cBti8dPGkC3sA=
github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ=
github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=

2
i18n/de.json
View file

@ -354,6 +354,8 @@
"settings.media.s3.secret": "AWS Access Secret",
"settings.media.s3.uploadExpiry": "Upload Ablaufdatum",
"settings.media.s3.uploadExpiryHelp": "(Optional) Zeit bis zum Ablauf (in Sekunden) für die generierte URL. Nur für private Buckets. (s, m, h, d für Sekunden, Minuten, Stunden, Tage).",
"settings.media.s3.url": "S3 backend URL",
"settings.media.s3.urlHelp": "Only change if using a custom S3 comptaible backend like Minio.",
"settings.media.title": "Medien Uploads",
"settings.media.upload.path": "Upload Pfad",
"settings.media.upload.pathHelp": "Pfad zum Upload Verzeichnis.",

2
i18n/en.json
View file

@ -354,6 +354,8 @@
"settings.media.s3.secret": "AWS access secret",
"settings.media.s3.uploadExpiry": "Upload expiry",
"settings.media.s3.uploadExpiryHelp": "(Optional) Specify TTL (in seconds) for the generated presigned URL. Only applicable for private buckets (s, m, h, d for seconds, minutes, hours, days).",
"settings.media.s3.url": "S3 backend URL",
"settings.media.s3.urlHelp": "Only change if using a custom S3 comptaible backend like Minio.",
"settings.media.title": "Media uploads",
"settings.media.upload.path": "Upload path",
"settings.media.upload.pathHelp": "Path to the directory where media will be uploaded.",

2
i18n/es.json
View file

@ -354,6 +354,8 @@
"settings.media.s3.secret": "Secrete de acceso a AWS (secret)",
"settings.media.s3.uploadExpiry": "Expiración de carga",
"settings.media.s3.uploadExpiryHelp": "(Opcional) TTL específico (en segundos) para la URL pre firmada generada. Solo es aplicable para contenedores privados (s, m, h, d para segundos, minutos, horas, días)",
"settings.media.s3.url": "S3 backend URL",
"settings.media.s3.urlHelp": "Only change if using a custom S3 comptaible backend like Minio.",
"settings.media.title": "Cargas de media",
"settings.media.upload.path": "Ruta de carga",
"settings.media.upload.pathHelp": "Ruta al directorio donde la media será cargada.",

2
i18n/fr.json
View file

@ -354,6 +354,8 @@
"settings.media.s3.secret": "Mot de passe d'accès AWS",
"settings.media.s3.uploadExpiry": "Durée de validité",
"settings.media.s3.uploadExpiryHelp": "(Facultatif) Spécifiez la durée de validité (en secondes) pour l'URL prédéfinie générée. Uniquement applicable pour les compartiments privés (s, m, h, d pour les secondes, minutes, heures, jours).",
"settings.media.s3.url": "S3 backend URL",
"settings.media.s3.urlHelp": "Only change if using a custom S3 comptaible backend like Minio.",
"settings.media.title": "Mise en ligne de fichiers",
"settings.media.upload.path": "Emplacement d'envoi des fichiers",
"settings.media.upload.pathHelp": "Chemin vers le répertoire où les médias seront mis en ligne",

2
i18n/it.json
View file

@ -354,6 +354,8 @@
"settings.media.s3.secret": "Accesso segreto AWS",
"settings.media.s3.uploadExpiry": "Caricamento scaduto",
"settings.media.s3.uploadExpiryHelp": "(Facoltativo) Specifica il TTL (in secondi) per l'URL predefinito generato. Applicabile solo per i buckets privati (s, m, h, d per i secondi, minuti, ore e giorni).",
"settings.media.s3.url": "S3 backend URL",
"settings.media.s3.urlHelp": "Only change if using a custom S3 comptaible backend like Minio.",
"settings.media.title": "Caricamento dei media",
"settings.media.upload.path": "Percorso del caricamento",
"settings.media.upload.pathHelp": "Percorso verso il repertorio dove i media saranno caricati.",

2
i18n/ml.json
View file

@ -354,6 +354,8 @@
"settings.media.s3.secret": "AWS പ്രവേശന രഹസ്യം",
"settings.media.s3.uploadExpiry": "അപ്ലോഡിന്റെ കാലാവധി",
"settings.media.s3.uploadExpiryHelp": "(ഐച്ഛികം) മുൻകൂട്ടി നിർമ്മിക്കുന്ന യൂ. ആർ. എല്ലിനുള്ള സെക്കന്റിലുള്ള TTL വ്യക്തമാക്കുക . സ്വകാര്യ ബക്കറ്റുകൾക്ക് മാത്രമേ ബാധകമാകൂ (s, m, h, d എന്നിവ യഥാക്രമം സെക്കന്റ്, മിനുട്ട്, മണിക്കൂർ, ദിവസങ്ങൾ എന്നിവയെ സൂചിപ്പിക്കുന്നു).",
"settings.media.s3.url": "S3 backend URL",
"settings.media.s3.urlHelp": "Only change if using a custom S3 comptaible backend like Minio.",
"settings.media.title": "മീഡിയാ അപ്ലോഡുകൾ",
"settings.media.upload.path": "അപ്ലോഡ് പാത്ത്",
"settings.media.upload.pathHelp": "മീഡിയ അപ്ലോഡ് ചെയ്യുന്നതിനുള്ള ഡയറക്ടറിയിലേക്കുള്ള പാത്ത്.",

2
i18n/pl.json
View file

@ -354,6 +354,8 @@
"settings.media.s3.secret": "Sekret dostępu AWS",
"settings.media.s3.uploadExpiry": "Wygaśnięcie przesyłania",
"settings.media.s3.uploadExpiryHelp": "(Opcjonalne) Zdefiniuj TTL (w sekundach) dla wygenerowanego podpisanego URL. Tylko dla prywatnych komór (bucketów) (s, m, h, d dla sekund, minut, godzin, dni).",
"settings.media.s3.url": "S3 backend URL",
"settings.media.s3.urlHelp": "Only change if using a custom S3 comptaible backend like Minio.",
"settings.media.title": "Wysyłka mediów",
"settings.media.upload.path": "Ścieżka do wysyłki",
"settings.media.upload.pathHelp": "Ścieżka do folderu do którego media będą wrzucane.",

2
i18n/pt-BR.json
View file

@ -354,6 +354,8 @@
"settings.media.s3.secret": "Segredo de acesso AWS",
"settings.media.s3.uploadExpiry": "Expiração do arquivo enviado",
"settings.media.s3.uploadExpiryHelp": "(Opcional) Especificar TTL (em segundos) para a URL pré-assinada gerada. Apenas aplicável para buckets privados (s, m, h, d para segundos, minutos, horas e dias).",
"settings.media.s3.url": "S3 backend URL",
"settings.media.s3.urlHelp": "Only change if using a custom S3 comptaible backend like Minio.",
"settings.media.title": "Envios de mídias",
"settings.media.upload.path": "Caminho de envio",
"settings.media.upload.pathHelp": "Caminho para o diretório onde a mídia será enviado.",

2
i18n/pt.json
View file

@ -354,6 +354,8 @@
"settings.media.s3.secret": "Segredo de acesso AWS",
"settings.media.s3.uploadExpiry": "Validade do upload",
"settings.media.s3.uploadExpiryHelp": "(Opcional) Especifica TTL (em segundos) para o URL pré-assinado gerado. Apenas aplicável a buckets privados (s, m, h, d para segundos, minutos, horas e dias).",
"settings.media.s3.url": "S3 backend URL",
"settings.media.s3.urlHelp": "Only change if using a custom S3 comptaible backend like Minio.",
"settings.media.title": "Upload de mídia",
"settings.media.upload.path": "Caminho de upload",
"settings.media.upload.pathHelp": "Caminho para a pasta onde será enviada a mídia.",

2
i18n/ru.json
View file

@ -354,6 +354,8 @@
"settings.media.s3.secret": "Секретаня фраза AWS",
"settings.media.s3.uploadExpiry": "Срок жизни выгрузки",
"settings.media.s3.uploadExpiryHelp": "(Необязательно) Укажите TTL (в секундах) сгенерированного подписанного URL. Применимо только для приватных bucket (s, m, h, d соответствует секундам, минутам, часам и дням).",
"settings.media.s3.url": "S3 backend URL",
"settings.media.s3.urlHelp": "Only change if using a custom S3 comptaible backend like Minio.",
"settings.media.title": "Выгрузки медиа",
"settings.media.upload.path": "Путь для выгрузок",
"settings.media.upload.pathHelp": "Путь до каталога, куда будут выгружаться медиа-файлы.",

2
i18n/tr.json
View file

@ -354,6 +354,8 @@
"settings.media.s3.secret": "AWS erişim şifresi(secret)",
"settings.media.s3.uploadExpiry": "Yükleme sona erme",
"settings.media.s3.uploadExpiryHelp": "(İsteğe bağlı) Oluşturulan önceden imzalanmış URL için TTL'yi (saniye cinsinden) belirtin. Yalnızca özel paketler için geçerlidir (saniye, dakika, saat, gün için s, m, h, d).",
"settings.media.s3.url": "S3 backend URL",
"settings.media.s3.urlHelp": "Only change if using a custom S3 comptaible backend like Minio.",
"settings.media.title": "Medya yüklemeleri",
"settings.media.upload.path": "Yükleme yolu",
"settings.media.upload.pathHelp": "Medyanın yükleneceği dizinin yolu.",

74
internal/media/providers/s3/s3.go
View file

@ -2,7 +2,6 @@ package s3
import (
"errors"
"fmt"
"io"
"strings"
"time"
@ -11,16 +10,14 @@ import (
"github.com/rhnvrm/simples3"
)
const amznS3PublicURL = "https://%s.s3.%s.amazonaws.com%s"
// Opts represents AWS S3 specific params
type Opts struct {
// Opt represents AWS S3 specific params
type Opt struct {
URL string `koanf:"url"`
AccessKey string `koanf:"aws_access_key_id"`
SecretKey string `koanf:"aws_secret_access_key"`
Region string `koanf:"aws_default_region"`
Bucket string `koanf:"bucket"`
BucketPath string `koanf:"bucket_path"`
BucketURL string `koanf:"bucket_url"`
BucketType string `koanf:"bucket_type"`
Expiry time.Duration `koanf:"expiry"`
}
@ -28,30 +25,36 @@ type Opts struct {
// Client implements `media.Store` for S3 provider
type Client struct {
s3 *simples3.S3
opts Opts
opts Opt
}
// NewS3Store initialises store for S3 provider. It takes in the AWS configuration
// and sets up the `simples3` client to interact with AWS APIs for all bucket operations.
func NewS3Store(opts Opts) (media.Store, error) {
var s3svc *simples3.S3
var err error
if opts.Region == "" {
return nil, errors.New("Invalid AWS Region specified. Please check `upload.s3` config")
func NewS3Store(opt Opt) (media.Store, error) {
var (
cl *simples3.S3
err error
)
if opt.URL == "" {
return nil, errors.New("Invalid AWS URL in settings.")
}
opt.URL = strings.TrimRight(opt.URL, "/")
// Use Access Key/Secret Key if specified in config.
if opts.AccessKey != "" && opts.SecretKey != "" {
s3svc = simples3.New(opts.Region, opts.AccessKey, opts.SecretKey)
if opt.AccessKey != "" && opt.SecretKey != "" {
cl = simples3.New(opt.Region, opt.AccessKey, opt.SecretKey)
} else {
// fallback to IAM role if no access key/secret key is provided.
s3svc, err = simples3.NewUsingIAM(opts.Region)
cl, err = simples3.NewUsingIAM(opt.Region)
if err != nil {
return nil, err
}
}
cl.SetEndpoint(opt.URL)
return &Client{
s3: s3svc,
opts: opts,
s3: cl,
opts: opt,
}, nil
}
@ -65,7 +68,7 @@ func (c *Client) Put(name string, cType string, file io.ReadSeeker) (string, err
Body: file,
// Paths inside the bucket should not start with /.
ObjectKey: strings.TrimPrefix(makeBucketPath(c.opts.BucketPath, name), "/"),
ObjectKey: c.makeBucketPath(name),
}
// Perform an upload.
if _, err := c.s3.FileUpload(upParams); err != nil {
@ -78,39 +81,42 @@ func (c *Client) Put(name string, cType string, file io.ReadSeeker) (string, err
func (c *Client) Get(name string) string {
// Generate a private S3 pre-signed URL if it's a private bucket.
if c.opts.BucketType == "private" {
url := c.s3.GeneratePresignedURL(simples3.PresignedInput{
u := c.s3.GeneratePresignedURL(simples3.PresignedInput{
Bucket: c.opts.Bucket,
ObjectKey: makeBucketPath(c.opts.BucketPath, name),
ObjectKey: c.makeBucketPath(name),
Method: "GET",
Timestamp: time.Now(),
ExpirySeconds: int(c.opts.Expiry.Seconds()),
})
return url
return u
}
// Generate a public S3 URL if it's a public bucket.
url := ""
if c.opts.BucketURL != "" {
url = c.opts.BucketURL + makeBucketPath(c.opts.BucketPath, name)
} else {
url = fmt.Sprintf(amznS3PublicURL, c.opts.Bucket, c.opts.Region,
makeBucketPath(c.opts.BucketPath, name))
}
return url
return c.makeFileURL(name)
}
// Delete accepts the filename of the object and deletes from S3.
func (c *Client) Delete(name string) error {
err := c.s3.FileDelete(simples3.DeleteInput{
Bucket: c.opts.Bucket,
ObjectKey: strings.TrimPrefix(makeBucketPath(c.opts.BucketPath, name), "/"),
ObjectKey: c.makeBucketPath(name),
})
return err
}
func makeBucketPath(bucketPath string, name string) string {
if bucketPath == "/" {
return "/" + name
// makeBucketPath returns the file path inside the bucket. The path should not
// start with a /.
func (c *Client) makeBucketPath(name string) string {
// If the path is root (/), return the filename without the preceding slash.
p := strings.TrimPrefix(strings.TrimSuffix(c.opts.BucketPath, "/"), "/")
if p == "" {
return name
}
return fmt.Sprintf("%s/%s", bucketPath, name)
// whatever/bucket/path/filename.jpg: No preceding slash.
return p + "/" + name
}
func (c *Client) makeFileURL(name string) string {
return c.opts.URL + "/" + c.opts.Bucket + "/" + c.makeBucketPath(name)
}

2
internal/migrations/v0.7.0.go
View file

@ -85,7 +85,7 @@ func V0_7_0(db *sqlx.DB, fs stuffbin.FileSystem, ko *koanf.Koanf) error {
('upload.filesystem.upload_uri', '"/uploads"'),
('upload.s3.aws_access_key_id', '""'),
('upload.s3.aws_secret_access_key', '""'),
('upload.s3.aws_default_region', '"ap-south-b"'),
('upload.s3.aws_default_region', '"ap-south-1"'),
('upload.s3.bucket', '""'),
('upload.s3.bucket_domain', '""'),
('upload.s3.bucket_path', '"/"'),

12
internal/migrations/v2.0.0.go
View file

@ -43,5 +43,17 @@ func V2_0_0(db *sqlx.DB, fs stuffbin.FileSystem, ko *koanf.Koanf) error {
return err
}
// S3 URL i snow a settings field. Prepare S3 URL based on region and bucket.
if _, err := db.Exec(`
WITH region AS (
SELECT value#>>'{}' AS value FROM settings WHERE key='upload.s3.aws_default_region'
), s3url AS (
SELECT FORMAT('https://s3.%s.amazonaws.com', (SELECT value FROM region)) AS value
)
INSERT INTO settings (key, value) VALUES ('upload.s3.url', TO_JSON((SELECT * FROM s3url))) ON CONFLICT DO NOTHING;`); err != nil {
return err
}
return nil
}

3
schema.sql
View file

@ -190,9 +190,10 @@ INSERT INTO settings (key, value) VALUES
('upload.provider', '"filesystem"'),
('upload.filesystem.upload_path', '"uploads"'),
('upload.filesystem.upload_uri', '"/uploads"'),
('upload.s3.url', '"https://ap-south-1.s3.amazonaws.com"'),
('upload.s3.aws_access_key_id', '""'),
('upload.s3.aws_secret_access_key', '""'),
('upload.s3.aws_default_region', '"ap-south-b"'),
('upload.s3.aws_default_region', '"ap-south-1"'),
('upload.s3.bucket', '""'),
('upload.s3.bucket_domain', '""'),
('upload.s3.bucket_path', '"/"'),