beenull/sftpgo-mirror
1
0
Fork 0
mirror of https://github.com/drakkan/sftpgo.git synced 2024-11-21 23:20:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

WebDAV: allow to define custom MIME type mappings

Browse source 
Fixes #1154

Signed-off-by: Nicola Murino <nicola.murino@gmail.com>
This commit is contained in:
Nicola Murino 2023-01-23 18:43:25 +01:00
parent 61199172d0
commit 2066ad7c83
No known key found for this signature in database
GPG key ID: 935D2952DEC4EECF
11 changed files with 183 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
docs/full-configuration.md
View file

@ -220,10 +220,16 @@ The configuration file contains the following sections:
- `options_passthrough`, boolean.
- `options_success_status`, integer.
- `allow_private_network`, boolean.
- `cache` struct containing cache configuration for the authenticated users.
- `enabled`, boolean, set to true to enable user caching. Default: true.
- `expiration_time`, integer. Expiration time, in minutes, for the cached users. 0 means unlimited. Default: 0.
- `max_size`, integer. Maximum number of users to cache. 0 means unlimited. Default: 50.
- `cache` struct containing cache configurations.
- `users`, cache configuration for the authenticated users.
- `expiration_time`, integer. Expiration time, in minutes, for the cached users. 0 means unlimited. Default: 0.
- `max_size`, integer. Maximum number of users to cache. 0 means unlimited. Default: 50.
- `mime_types`, cache configuration for mime types.
- `enabled`, boolean, set to true to enable mime types caching. Default: `true`.
- `max_size`, integer. Maximum number of mime types to cache. 0 means no cache. Default: 1000.
- `custom_mappings`, additional mime types mapping. This is a platform independet way to add few additional mappings. You can set a limited number of mappings here, if you want to add a large list use the method provided by the OS of your choice. List of struct, each struct has the following fields:
- `ext`, string, file extension including the dot, for example `.json`
- `mime`, string, mime type, for example `application/json`
</details>
<details><summary><font size=4>Data Provider</font></summary>

4
go.mod
View file

@ -20,7 +20,7 @@ require (
github.com/bmatcuk/doublestar/v4 v4.6.0
github.com/cockroachdb/cockroach-go/v2 v2.2.20
github.com/coreos/go-oidc/v3 v3.5.0
github.com/drakkan/webdav v0.0.0-20221101181759-17ed21f9337b
github.com/drakkan/webdav v0.0.0-20230123134431-a95c027a0038
github.com/eikenb/pipeat v0.0.0-20210730190139-06b3e6902001
github.com/fclairamb/ftpserverlib v0.20.1-0.20230104020606-0b1a04eec221
github.com/fclairamb/go-log v0.4.1
@ -104,7 +104,7 @@ require (
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
github.com/fatih/color v1.13.0 // indirect
github.com/fatih/color v1.14.1 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/go-jose/go-jose/v3 v3.0.0 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect

7
go.sum
View file

@ -851,8 +851,8 @@ github.com/drakkan/crypto v0.0.0-20230106095953-5417b4dfde62 h1:1Bk+GbTbF1PBu0id
github.com/drakkan/crypto v0.0.0-20230106095953-5417b4dfde62/go.mod h1:eekSq7nI5pP2ZldL4867reOp0VL9TOfTaZa0DydSYk4=
github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9 h1:LPH1dEblAOO/LoG7yHPMtBLXhQmjaga91/DDjWk9jWA=
github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9/go.mod h1:2lmrmq866uF2tnje75wQHzmPXhmSWUt7Gyx2vgK1RCU=
github.com/drakkan/webdav v0.0.0-20221101181759-17ed21f9337b h1:B9z7XyDoVxLO4yEvnXgdvZ+0Uw9NA1qdD4KTSGmKcoQ=
github.com/drakkan/webdav v0.0.0-20221101181759-17ed21f9337b/go.mod h1:8opebuqUyBXrvl7Vo/S1Zzl9U0G1X2Ceud440eVuhUE=
github.com/drakkan/webdav v0.0.0-20230123134431-a95c027a0038 h1:vQe1F4uoOg7fmli8L/MyzbeAhf6SfM3M1bTGcgHscAw=
github.com/drakkan/webdav v0.0.0-20230123134431-a95c027a0038/go.mod h1:8opebuqUyBXrvl7Vo/S1Zzl9U0G1X2Ceud440eVuhUE=
github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
@ -887,8 +887,9 @@ github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQL
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w=
github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg=
github.com/fclairamb/ftpserverlib v0.20.1-0.20230104020606-0b1a04eec221 h1:oIEBdcX1yNS5F+rk0xaDXMkwu9cT6+YSBEih45Wptec=
github.com/fclairamb/ftpserverlib v0.20.1-0.20230104020606-0b1a04eec221/go.mod h1:2PS2QXGtruTtfUszbKGOuuWhDiK5u/GD9DK2DdAW+S8=
github.com/fclairamb/go-log v0.4.1 h1:rLtdSG9x2pK41AIAnE8WYpl05xBJfw1ZyYxZaXFcBsM=

28
internal/config/config.go
View file

@ -309,8 +309,9 @@ func Init() {
MaxSize: 50,
},
MimeTypes: webdavd.MimeCacheConfig{
Enabled: true,
MaxSize: 1000,
Enabled: true,
MaxSize: 1000,
CustomMappings: nil,
},
},
},
@ -732,6 +733,7 @@ func LoadConfig(configDir, configFile string) error {
}
// viper only supports slice of strings from env vars, so we use our custom method
loadBindingsFromEnv()
loadWebDAVCacheMappingsFromEnv()
resetInvalidConfigs()
logger.Debug(logSender, "", "config file used: '%#v', config loaded: %+v", viper.ConfigFileUsed(), getRedactedGlobalConf())
return nil
@ -1249,6 +1251,27 @@ func getWebDAVDBindingProxyConfigsFromEnv(idx int, binding *webdavd.Binding) boo
return isSet
}
func loadWebDAVCacheMappingsFromEnv() []webdavd.CustomMimeMapping {
for idx := 0; idx < 30; idx++ {
ext, extOK := os.LookupEnv(fmt.Sprintf("SFTPGO_WEBDAVD__CACHE__MIME_TYPES__CUSTOM_MAPPINGS__%d__EXT", idx))
mime, mimeOK := os.LookupEnv(fmt.Sprintf("SFTPGO_WEBDAVD__CACHE__MIME_TYPES__CUSTOM_MAPPINGS__%d__MIME", idx))
if extOK && mimeOK {
if len(globalConf.WebDAVD.Cache.MimeTypes.CustomMappings) > idx {
globalConf.WebDAVD.Cache.MimeTypes.CustomMappings[idx].Ext = ext
globalConf.WebDAVD.Cache.MimeTypes.CustomMappings[idx].Mime = mime
} else {
globalConf.WebDAVD.Cache.MimeTypes.CustomMappings = append(globalConf.WebDAVD.Cache.MimeTypes.CustomMappings,
webdavd.CustomMimeMapping{
Ext: ext,
Mime: mime,
})
}
}
}
return globalConf.WebDAVD.Cache.MimeTypes.CustomMappings
}
func getWebDAVDBindingFromEnv(idx int) {
binding := defaultWebDAVDBinding
if len(globalConf.WebDAVD.Bindings) > idx {
@ -2010,6 +2033,7 @@ func setViperDefaults() {
viper.SetDefault("webdavd.cache.users.max_size", globalConf.WebDAVD.Cache.Users.MaxSize)
viper.SetDefault("webdavd.cache.mime_types.enabled", globalConf.WebDAVD.Cache.MimeTypes.Enabled)
viper.SetDefault("webdavd.cache.mime_types.max_size", globalConf.WebDAVD.Cache.MimeTypes.MaxSize)
viper.SetDefault("webdavd.cache.mime_types.custom_mappings", globalConf.WebDAVD.Cache.MimeTypes.CustomMappings)
viper.SetDefault("data_provider.driver", globalConf.ProviderConf.Driver)
viper.SetDefault("data_provider.name", globalConf.ProviderConf.Name)
viper.SetDefault("data_provider.host", globalConf.ProviderConf.Host)

67
internal/config/config_test.go
View file

@ -38,6 +38,7 @@ import (
"github.com/drakkan/sftpgo/v2/internal/sftpd"
"github.com/drakkan/sftpgo/v2/internal/smtp"
"github.com/drakkan/sftpgo/v2/internal/util"
"github.com/drakkan/sftpgo/v2/internal/webdavd"
)
const (
@ -1031,6 +1032,72 @@ func TestFTPDBindingsFromEnv(t *testing.T) {
require.Equal(t, "cert.key", bindings[1].CertificateKeyFile)
}
func TestWebDAVMimeCache(t *testing.T) {
reset()
err := config.LoadConfig(configDir, "")
assert.NoError(t, err)
webdavdConf := config.GetWebDAVDConfig()
webdavdConf.Cache.MimeTypes.CustomMappings = []webdavd.CustomMimeMapping{
{
Ext: ".custom",
Mime: "application/custom",
},
}
cfg := map[string]any{
"webdavd": webdavdConf,
}
data, err := json.Marshal(cfg)
assert.NoError(t, err)
confName := tempConfigName + ".json"
configFilePath := filepath.Join(configDir, confName)
err = os.WriteFile(configFilePath, data, 0666)
assert.NoError(t, err)
reset()
err = config.LoadConfig(configDir, confName)
assert.NoError(t, err)
mappings := config.GetWebDAVDConfig().Cache.MimeTypes.CustomMappings
if assert.Len(t, mappings, 1) {
assert.Equal(t, ".custom", mappings[0].Ext)
assert.Equal(t, "application/custom", mappings[0].Mime)
}
// now add from env
os.Setenv("SFTPGO_WEBDAVD__CACHE__MIME_TYPES__CUSTOM_MAPPINGS__1__EXT", ".custom1")
os.Setenv("SFTPGO_WEBDAVD__CACHE__MIME_TYPES__CUSTOM_MAPPINGS__1__MIME", "application/custom1")
t.Cleanup(func() {
os.Unsetenv("SFTPGO_WEBDAVD__CACHE__MIME_TYPES__CUSTOM_MAPPINGS__0__EXT")
os.Unsetenv("SFTPGO_WEBDAVD__CACHE__MIME_TYPES__CUSTOM_MAPPINGS__0__MIME")
os.Unsetenv("SFTPGO_WEBDAVD__CACHE__MIME_TYPES__CUSTOM_MAPPINGS__1__EXT")
os.Unsetenv("SFTPGO_WEBDAVD__CACHE__MIME_TYPES__CUSTOM_MAPPINGS__1__MIME")
})
reset()
err = config.LoadConfig(configDir, confName)
assert.NoError(t, err)
mappings = config.GetWebDAVDConfig().Cache.MimeTypes.CustomMappings
if assert.Len(t, mappings, 2) {
assert.Equal(t, ".custom", mappings[0].Ext)
assert.Equal(t, "application/custom", mappings[0].Mime)
assert.Equal(t, ".custom1", mappings[1].Ext)
assert.Equal(t, "application/custom1", mappings[1].Mime)
}
// override from env
os.Setenv("SFTPGO_WEBDAVD__CACHE__MIME_TYPES__CUSTOM_MAPPINGS__0__EXT", ".custom0")
os.Setenv("SFTPGO_WEBDAVD__CACHE__MIME_TYPES__CUSTOM_MAPPINGS__0__MIME", "application/custom0")
reset()
err = config.LoadConfig(configDir, confName)
assert.NoError(t, err)
mappings = config.GetWebDAVDConfig().Cache.MimeTypes.CustomMappings
if assert.Len(t, mappings, 2) {
assert.Equal(t, ".custom0", mappings[0].Ext)
assert.Equal(t, "application/custom0", mappings[0].Mime)
assert.Equal(t, ".custom1", mappings[1].Ext)
assert.Equal(t, "application/custom1", mappings[1].Mime)
}
err = os.Remove(configFilePath)
assert.NoError(t, err)
}
func TestWebDAVBindingsFromEnv(t *testing.T) {
reset()

3
internal/webdavd/file.go
View file

@ -84,6 +84,9 @@ type webDavFileInfo struct {
// ContentType implements webdav.ContentTyper interface
func (fi *webDavFileInfo) ContentType(ctx context.Context) (string, error) {
extension := path.Ext(fi.virtualPath)
if ctype, ok := customMimeTypeMapping[extension]; ok {
return ctype, nil
}
if extension == "" || extension == ".dat" {
return "application/octet-stream", nil
}

12
internal/webdavd/internal_test.go
View file

@ -712,6 +712,18 @@ func TestContentType(t *testing.T) {
assert.NoError(t, err)
}
baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath, testFilePath, testFile+".sftpgo",
common.TransferDownload, 0, 0, 0, 0, false, fs, dataprovider.TransferQuota{})
fs = newMockOsFs(false, fs.ConnectionID(), user.GetHomeDir(), nil, os.ErrInvalid)
davFile = newWebDavFile(baseTransfer, nil, nil)
davFile.Fs = fs
fi, err = davFile.Stat()
if assert.NoError(t, err) {
ctype, err := fi.(*webDavFileInfo).ContentType(ctx)
assert.NoError(t, err)
assert.Equal(t, "application/sftpgo", ctype)
}
baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath, testFilePath, testFile+".unknown2",
common.TransferDownload, 0, 0, 0, 0, false, fs, dataprovider.TransferQuota{})
fs = newMockOsFs(false, fs.ConnectionID(), user.GetHomeDir(), nil, os.ErrInvalid)

5
internal/webdavd/mimecache.go
View file

@ -22,7 +22,10 @@ type mimeCache struct {
mimeTypes map[string]string
}
var mimeTypeCache mimeCache
var (
mimeTypeCache mimeCache
customMimeTypeMapping map[string]string
)
func (c *mimeCache) addMimeToCache(key, value string) {
c.Lock()

20
internal/webdavd/webdavd.go
View file

@ -64,6 +64,12 @@ type CorsConfig struct {
AllowPrivateNetwork bool `json:"allow_private_network" mapstructure:"allow_private_network"`
}
// CustomMimeMapping defines additional, user defined mime mappings
type CustomMimeMapping struct {
Ext string `json:"ext" mapstructure:"ext"`
Mime string `json:"mime" mapstructure:"mime"`
}
// UsersCacheConfig defines the cache configuration for users
type UsersCacheConfig struct {
ExpirationTime int `json:"expiration_time" mapstructure:"expiration_time"`
@ -72,8 +78,9 @@ type UsersCacheConfig struct {
// MimeCacheConfig defines the cache configuration for mime types
type MimeCacheConfig struct {
Enabled bool `json:"enabled" mapstructure:"enabled"`
MaxSize int `json:"max_size" mapstructure:"max_size"`
Enabled bool `json:"enabled" mapstructure:"enabled"`
MaxSize int `json:"max_size" mapstructure:"max_size"`
CustomMappings []CustomMimeMapping `json:"custom_mappings" mapstructure:"custom_mappings"`
}
// Cache configuration
@ -228,7 +235,16 @@ func (c *Configuration) Initialize(configDir string) error {
}
if !c.Cache.MimeTypes.Enabled {
mimeTypeCache.maxSize = 0
} else {
customMimeTypeMapping = make(map[string]string)
for _, m := range c.Cache.MimeTypes.CustomMappings {
if m.Mime != "" {
logger.Debug(logSender, "", "adding custom mime mapping for extension %q, mime type %q", m.Ext, m.Mime)
customMimeTypeMapping[m.Ext] = m.Mime
}
}
}
if !c.ShouldBind() {
return common.ErrNoBinding
}

35
internal/webdavd/webdavd_test.go
View file

@ -272,6 +272,8 @@ func TestMain(m *testing.M) {
os.Setenv("SFTPGO_COMMON__ALLOW_SELF_CONNECTIONS", "1")
os.Setenv("SFTPGO_DEFAULT_ADMIN_USERNAME", "admin")
os.Setenv("SFTPGO_DEFAULT_ADMIN_PASSWORD", "password")
os.Setenv("SFTPGO_WEBDAVD__CACHE__MIME_TYPES__CUSTOM_MAPPINGS__0__EXT", ".sftpgo")
os.Setenv("SFTPGO_WEBDAVD__CACHE__MIME_TYPES__CUSTOM_MAPPINGS__0__MIME", "application/sftpgo")
err := config.LoadConfig(configDir, "")
if err != nil {
logger.ErrorToConsole("error loading configuration: %v", err)
@ -2226,6 +2228,39 @@ func TestBytesRangeRequests(t *testing.T) {
assert.NoError(t, err)
}
func TestContentTypeGET(t *testing.T) {
user, _, err := httpdtest.AddUser(getTestUser(), http.StatusCreated)
assert.NoError(t, err)
testFilePath := filepath.Join(homeBasePath, testFileName)
testFileSize := int64(64)
err = createTestFile(testFilePath, testFileSize)
assert.NoError(t, err)
client := getWebDavClient(user, true, nil)
err = uploadFileWithRawClient(testFilePath, testFileName+".sftpgo", user.Username, defaultPassword,
true, testFileSize, client)
assert.NoError(t, err)
remotePath := fmt.Sprintf("http://%v/%v", webDavServerAddr, testFileName+".sftpgo")
req, err := http.NewRequest(http.MethodGet, remotePath, nil)
if assert.NoError(t, err) {
httpClient := httpclient.GetHTTPClient()
req.SetBasicAuth(user.Username, defaultPassword)
resp, err := httpClient.Do(req)
if assert.NoError(t, err) {
defer resp.Body.Close()
assert.Equal(t, http.StatusOK, resp.StatusCode)
assert.Equal(t, "application/sftpgo", resp.Header.Get("Content-Type"))
}
}
err = os.Remove(testFilePath)
assert.NoError(t, err)
_, err = httpdtest.RemoveUser(user, http.StatusOK)
assert.NoError(t, err)
err = os.RemoveAll(user.GetHomeDir())
assert.NoError(t, err)
}
func TestHEAD(t *testing.T) {
u := getTestUser()
user, _, err := httpdtest.AddUser(u, http.StatusCreated)

3
sftpgo.json
View file

@ -181,7 +181,8 @@
},
"mime_types": {
"enabled": true,
"max_size": 1000
"max_size": 1000,
"custom_mappings": []
}
}
},