From 2066ad7c83853ef2945348a233d890cbb02b55d1 Mon Sep 17 00:00:00 2001 From: Nicola Murino Date: Mon, 23 Jan 2023 18:43:25 +0100 Subject: [PATCH] WebDAV: allow to define custom MIME type mappings Fixes #1154 Signed-off-by: Nicola Murino --- docs/full-configuration.md | 14 +++++-- go.mod | 4 +- go.sum | 7 ++-- internal/config/config.go | 28 ++++++++++++- internal/config/config_test.go | 67 +++++++++++++++++++++++++++++++ internal/webdavd/file.go | 3 ++ internal/webdavd/internal_test.go | 12 ++++++ internal/webdavd/mimecache.go | 5 ++- internal/webdavd/webdavd.go | 20 ++++++++- internal/webdavd/webdavd_test.go | 35 ++++++++++++++++ sftpgo.json | 3 +- 11 files changed, 183 insertions(+), 15 deletions(-) diff --git a/docs/full-configuration.md b/docs/full-configuration.md index a9ff5cba..edab453a 100644 --- a/docs/full-configuration.md +++ b/docs/full-configuration.md @@ -220,10 +220,16 @@ The configuration file contains the following sections: - `options_passthrough`, boolean. - `options_success_status`, integer. - `allow_private_network`, boolean. - - `cache` struct containing cache configuration for the authenticated users. - - `enabled`, boolean, set to true to enable user caching. Default: true. - - `expiration_time`, integer. Expiration time, in minutes, for the cached users. 0 means unlimited. Default: 0. - - `max_size`, integer. Maximum number of users to cache. 0 means unlimited. Default: 50. + - `cache` struct containing cache configurations. + - `users`, cache configuration for the authenticated users. + - `expiration_time`, integer. Expiration time, in minutes, for the cached users. 0 means unlimited. Default: 0. + - `max_size`, integer. Maximum number of users to cache. 0 means unlimited. Default: 50. + - `mime_types`, cache configuration for mime types. + - `enabled`, boolean, set to true to enable mime types caching. Default: `true`. + - `max_size`, integer. Maximum number of mime types to cache. 0 means no cache. Default: 1000. + - `custom_mappings`, additional mime types mapping. This is a platform independet way to add few additional mappings. You can set a limited number of mappings here, if you want to add a large list use the method provided by the OS of your choice. List of struct, each struct has the following fields: + - `ext`, string, file extension including the dot, for example `.json` + - `mime`, string, mime type, for example `application/json`
Data Provider diff --git a/go.mod b/go.mod index 5253061d..2a191009 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/bmatcuk/doublestar/v4 v4.6.0 github.com/cockroachdb/cockroach-go/v2 v2.2.20 github.com/coreos/go-oidc/v3 v3.5.0 - github.com/drakkan/webdav v0.0.0-20221101181759-17ed21f9337b + github.com/drakkan/webdav v0.0.0-20230123134431-a95c027a0038 github.com/eikenb/pipeat v0.0.0-20210730190139-06b3e6902001 github.com/fclairamb/ftpserverlib v0.20.1-0.20230104020606-0b1a04eec221 github.com/fclairamb/go-log v0.4.1 @@ -104,7 +104,7 @@ require ( github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect - github.com/fatih/color v1.13.0 // indirect + github.com/fatih/color v1.14.1 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-jose/go-jose/v3 v3.0.0 // indirect github.com/go-ole/go-ole v1.2.6 // indirect diff --git a/go.sum b/go.sum index 4e596fce..c5a7d619 100644 --- a/go.sum +++ b/go.sum @@ -851,8 +851,8 @@ github.com/drakkan/crypto v0.0.0-20230106095953-5417b4dfde62 h1:1Bk+GbTbF1PBu0id github.com/drakkan/crypto v0.0.0-20230106095953-5417b4dfde62/go.mod h1:eekSq7nI5pP2ZldL4867reOp0VL9TOfTaZa0DydSYk4= github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9 h1:LPH1dEblAOO/LoG7yHPMtBLXhQmjaga91/DDjWk9jWA= github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9/go.mod h1:2lmrmq866uF2tnje75wQHzmPXhmSWUt7Gyx2vgK1RCU= -github.com/drakkan/webdav v0.0.0-20221101181759-17ed21f9337b h1:B9z7XyDoVxLO4yEvnXgdvZ+0Uw9NA1qdD4KTSGmKcoQ= -github.com/drakkan/webdav v0.0.0-20221101181759-17ed21f9337b/go.mod h1:8opebuqUyBXrvl7Vo/S1Zzl9U0G1X2Ceud440eVuhUE= +github.com/drakkan/webdav v0.0.0-20230123134431-a95c027a0038 h1:vQe1F4uoOg7fmli8L/MyzbeAhf6SfM3M1bTGcgHscAw= +github.com/drakkan/webdav v0.0.0-20230123134431-a95c027a0038/go.mod h1:8opebuqUyBXrvl7Vo/S1Zzl9U0G1X2Ceud440eVuhUE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= @@ -887,8 +887,9 @@ github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQL github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= -github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= +github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= github.com/fclairamb/ftpserverlib v0.20.1-0.20230104020606-0b1a04eec221 h1:oIEBdcX1yNS5F+rk0xaDXMkwu9cT6+YSBEih45Wptec= github.com/fclairamb/ftpserverlib v0.20.1-0.20230104020606-0b1a04eec221/go.mod h1:2PS2QXGtruTtfUszbKGOuuWhDiK5u/GD9DK2DdAW+S8= github.com/fclairamb/go-log v0.4.1 h1:rLtdSG9x2pK41AIAnE8WYpl05xBJfw1ZyYxZaXFcBsM= diff --git a/internal/config/config.go b/internal/config/config.go index de6c108e..ccd9aa1a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -309,8 +309,9 @@ func Init() { MaxSize: 50, }, MimeTypes: webdavd.MimeCacheConfig{ - Enabled: true, - MaxSize: 1000, + Enabled: true, + MaxSize: 1000, + CustomMappings: nil, }, }, }, @@ -732,6 +733,7 @@ func LoadConfig(configDir, configFile string) error { } // viper only supports slice of strings from env vars, so we use our custom method loadBindingsFromEnv() + loadWebDAVCacheMappingsFromEnv() resetInvalidConfigs() logger.Debug(logSender, "", "config file used: '%#v', config loaded: %+v", viper.ConfigFileUsed(), getRedactedGlobalConf()) return nil @@ -1249,6 +1251,27 @@ func getWebDAVDBindingProxyConfigsFromEnv(idx int, binding *webdavd.Binding) boo return isSet } +func loadWebDAVCacheMappingsFromEnv() []webdavd.CustomMimeMapping { + for idx := 0; idx < 30; idx++ { + ext, extOK := os.LookupEnv(fmt.Sprintf("SFTPGO_WEBDAVD__CACHE__MIME_TYPES__CUSTOM_MAPPINGS__%d__EXT", idx)) + mime, mimeOK := os.LookupEnv(fmt.Sprintf("SFTPGO_WEBDAVD__CACHE__MIME_TYPES__CUSTOM_MAPPINGS__%d__MIME", idx)) + if extOK && mimeOK { + if len(globalConf.WebDAVD.Cache.MimeTypes.CustomMappings) > idx { + globalConf.WebDAVD.Cache.MimeTypes.CustomMappings[idx].Ext = ext + globalConf.WebDAVD.Cache.MimeTypes.CustomMappings[idx].Mime = mime + } else { + globalConf.WebDAVD.Cache.MimeTypes.CustomMappings = append(globalConf.WebDAVD.Cache.MimeTypes.CustomMappings, + webdavd.CustomMimeMapping{ + Ext: ext, + Mime: mime, + }) + } + } + } + + return globalConf.WebDAVD.Cache.MimeTypes.CustomMappings +} + func getWebDAVDBindingFromEnv(idx int) { binding := defaultWebDAVDBinding if len(globalConf.WebDAVD.Bindings) > idx { @@ -2010,6 +2033,7 @@ func setViperDefaults() { viper.SetDefault("webdavd.cache.users.max_size", globalConf.WebDAVD.Cache.Users.MaxSize) viper.SetDefault("webdavd.cache.mime_types.enabled", globalConf.WebDAVD.Cache.MimeTypes.Enabled) viper.SetDefault("webdavd.cache.mime_types.max_size", globalConf.WebDAVD.Cache.MimeTypes.MaxSize) + viper.SetDefault("webdavd.cache.mime_types.custom_mappings", globalConf.WebDAVD.Cache.MimeTypes.CustomMappings) viper.SetDefault("data_provider.driver", globalConf.ProviderConf.Driver) viper.SetDefault("data_provider.name", globalConf.ProviderConf.Name) viper.SetDefault("data_provider.host", globalConf.ProviderConf.Host) diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 0d242aa9..4e634bf4 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -38,6 +38,7 @@ import ( "github.com/drakkan/sftpgo/v2/internal/sftpd" "github.com/drakkan/sftpgo/v2/internal/smtp" "github.com/drakkan/sftpgo/v2/internal/util" + "github.com/drakkan/sftpgo/v2/internal/webdavd" ) const ( @@ -1031,6 +1032,72 @@ func TestFTPDBindingsFromEnv(t *testing.T) { require.Equal(t, "cert.key", bindings[1].CertificateKeyFile) } +func TestWebDAVMimeCache(t *testing.T) { + reset() + + err := config.LoadConfig(configDir, "") + assert.NoError(t, err) + webdavdConf := config.GetWebDAVDConfig() + webdavdConf.Cache.MimeTypes.CustomMappings = []webdavd.CustomMimeMapping{ + { + Ext: ".custom", + Mime: "application/custom", + }, + } + cfg := map[string]any{ + "webdavd": webdavdConf, + } + data, err := json.Marshal(cfg) + assert.NoError(t, err) + confName := tempConfigName + ".json" + configFilePath := filepath.Join(configDir, confName) + err = os.WriteFile(configFilePath, data, 0666) + assert.NoError(t, err) + + reset() + err = config.LoadConfig(configDir, confName) + assert.NoError(t, err) + mappings := config.GetWebDAVDConfig().Cache.MimeTypes.CustomMappings + if assert.Len(t, mappings, 1) { + assert.Equal(t, ".custom", mappings[0].Ext) + assert.Equal(t, "application/custom", mappings[0].Mime) + } + // now add from env + os.Setenv("SFTPGO_WEBDAVD__CACHE__MIME_TYPES__CUSTOM_MAPPINGS__1__EXT", ".custom1") + os.Setenv("SFTPGO_WEBDAVD__CACHE__MIME_TYPES__CUSTOM_MAPPINGS__1__MIME", "application/custom1") + t.Cleanup(func() { + os.Unsetenv("SFTPGO_WEBDAVD__CACHE__MIME_TYPES__CUSTOM_MAPPINGS__0__EXT") + os.Unsetenv("SFTPGO_WEBDAVD__CACHE__MIME_TYPES__CUSTOM_MAPPINGS__0__MIME") + os.Unsetenv("SFTPGO_WEBDAVD__CACHE__MIME_TYPES__CUSTOM_MAPPINGS__1__EXT") + os.Unsetenv("SFTPGO_WEBDAVD__CACHE__MIME_TYPES__CUSTOM_MAPPINGS__1__MIME") + }) + reset() + err = config.LoadConfig(configDir, confName) + assert.NoError(t, err) + mappings = config.GetWebDAVDConfig().Cache.MimeTypes.CustomMappings + if assert.Len(t, mappings, 2) { + assert.Equal(t, ".custom", mappings[0].Ext) + assert.Equal(t, "application/custom", mappings[0].Mime) + assert.Equal(t, ".custom1", mappings[1].Ext) + assert.Equal(t, "application/custom1", mappings[1].Mime) + } + // override from env + os.Setenv("SFTPGO_WEBDAVD__CACHE__MIME_TYPES__CUSTOM_MAPPINGS__0__EXT", ".custom0") + os.Setenv("SFTPGO_WEBDAVD__CACHE__MIME_TYPES__CUSTOM_MAPPINGS__0__MIME", "application/custom0") + reset() + err = config.LoadConfig(configDir, confName) + assert.NoError(t, err) + mappings = config.GetWebDAVDConfig().Cache.MimeTypes.CustomMappings + if assert.Len(t, mappings, 2) { + assert.Equal(t, ".custom0", mappings[0].Ext) + assert.Equal(t, "application/custom0", mappings[0].Mime) + assert.Equal(t, ".custom1", mappings[1].Ext) + assert.Equal(t, "application/custom1", mappings[1].Mime) + } + err = os.Remove(configFilePath) + assert.NoError(t, err) +} + func TestWebDAVBindingsFromEnv(t *testing.T) { reset() diff --git a/internal/webdavd/file.go b/internal/webdavd/file.go index 382c70ea..225b19c5 100644 --- a/internal/webdavd/file.go +++ b/internal/webdavd/file.go @@ -84,6 +84,9 @@ type webDavFileInfo struct { // ContentType implements webdav.ContentTyper interface func (fi *webDavFileInfo) ContentType(ctx context.Context) (string, error) { extension := path.Ext(fi.virtualPath) + if ctype, ok := customMimeTypeMapping[extension]; ok { + return ctype, nil + } if extension == "" || extension == ".dat" { return "application/octet-stream", nil } diff --git a/internal/webdavd/internal_test.go b/internal/webdavd/internal_test.go index ef49401d..7d1d9169 100644 --- a/internal/webdavd/internal_test.go +++ b/internal/webdavd/internal_test.go @@ -712,6 +712,18 @@ func TestContentType(t *testing.T) { assert.NoError(t, err) } + baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath, testFilePath, testFile+".sftpgo", + common.TransferDownload, 0, 0, 0, 0, false, fs, dataprovider.TransferQuota{}) + fs = newMockOsFs(false, fs.ConnectionID(), user.GetHomeDir(), nil, os.ErrInvalid) + davFile = newWebDavFile(baseTransfer, nil, nil) + davFile.Fs = fs + fi, err = davFile.Stat() + if assert.NoError(t, err) { + ctype, err := fi.(*webDavFileInfo).ContentType(ctx) + assert.NoError(t, err) + assert.Equal(t, "application/sftpgo", ctype) + } + baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath, testFilePath, testFile+".unknown2", common.TransferDownload, 0, 0, 0, 0, false, fs, dataprovider.TransferQuota{}) fs = newMockOsFs(false, fs.ConnectionID(), user.GetHomeDir(), nil, os.ErrInvalid) diff --git a/internal/webdavd/mimecache.go b/internal/webdavd/mimecache.go index 907e472b..6a228906 100644 --- a/internal/webdavd/mimecache.go +++ b/internal/webdavd/mimecache.go @@ -22,7 +22,10 @@ type mimeCache struct { mimeTypes map[string]string } -var mimeTypeCache mimeCache +var ( + mimeTypeCache mimeCache + customMimeTypeMapping map[string]string +) func (c *mimeCache) addMimeToCache(key, value string) { c.Lock() diff --git a/internal/webdavd/webdavd.go b/internal/webdavd/webdavd.go index 6c1dac70..ff3c14f7 100644 --- a/internal/webdavd/webdavd.go +++ b/internal/webdavd/webdavd.go @@ -64,6 +64,12 @@ type CorsConfig struct { AllowPrivateNetwork bool `json:"allow_private_network" mapstructure:"allow_private_network"` } +// CustomMimeMapping defines additional, user defined mime mappings +type CustomMimeMapping struct { + Ext string `json:"ext" mapstructure:"ext"` + Mime string `json:"mime" mapstructure:"mime"` +} + // UsersCacheConfig defines the cache configuration for users type UsersCacheConfig struct { ExpirationTime int `json:"expiration_time" mapstructure:"expiration_time"` @@ -72,8 +78,9 @@ type UsersCacheConfig struct { // MimeCacheConfig defines the cache configuration for mime types type MimeCacheConfig struct { - Enabled bool `json:"enabled" mapstructure:"enabled"` - MaxSize int `json:"max_size" mapstructure:"max_size"` + Enabled bool `json:"enabled" mapstructure:"enabled"` + MaxSize int `json:"max_size" mapstructure:"max_size"` + CustomMappings []CustomMimeMapping `json:"custom_mappings" mapstructure:"custom_mappings"` } // Cache configuration @@ -228,7 +235,16 @@ func (c *Configuration) Initialize(configDir string) error { } if !c.Cache.MimeTypes.Enabled { mimeTypeCache.maxSize = 0 + } else { + customMimeTypeMapping = make(map[string]string) + for _, m := range c.Cache.MimeTypes.CustomMappings { + if m.Mime != "" { + logger.Debug(logSender, "", "adding custom mime mapping for extension %q, mime type %q", m.Ext, m.Mime) + customMimeTypeMapping[m.Ext] = m.Mime + } + } } + if !c.ShouldBind() { return common.ErrNoBinding } diff --git a/internal/webdavd/webdavd_test.go b/internal/webdavd/webdavd_test.go index ad1de4f0..435d4a05 100644 --- a/internal/webdavd/webdavd_test.go +++ b/internal/webdavd/webdavd_test.go @@ -272,6 +272,8 @@ func TestMain(m *testing.M) { os.Setenv("SFTPGO_COMMON__ALLOW_SELF_CONNECTIONS", "1") os.Setenv("SFTPGO_DEFAULT_ADMIN_USERNAME", "admin") os.Setenv("SFTPGO_DEFAULT_ADMIN_PASSWORD", "password") + os.Setenv("SFTPGO_WEBDAVD__CACHE__MIME_TYPES__CUSTOM_MAPPINGS__0__EXT", ".sftpgo") + os.Setenv("SFTPGO_WEBDAVD__CACHE__MIME_TYPES__CUSTOM_MAPPINGS__0__MIME", "application/sftpgo") err := config.LoadConfig(configDir, "") if err != nil { logger.ErrorToConsole("error loading configuration: %v", err) @@ -2226,6 +2228,39 @@ func TestBytesRangeRequests(t *testing.T) { assert.NoError(t, err) } +func TestContentTypeGET(t *testing.T) { + user, _, err := httpdtest.AddUser(getTestUser(), http.StatusCreated) + assert.NoError(t, err) + + testFilePath := filepath.Join(homeBasePath, testFileName) + testFileSize := int64(64) + err = createTestFile(testFilePath, testFileSize) + assert.NoError(t, err) + client := getWebDavClient(user, true, nil) + err = uploadFileWithRawClient(testFilePath, testFileName+".sftpgo", user.Username, defaultPassword, + true, testFileSize, client) + assert.NoError(t, err) + remotePath := fmt.Sprintf("http://%v/%v", webDavServerAddr, testFileName+".sftpgo") + req, err := http.NewRequest(http.MethodGet, remotePath, nil) + if assert.NoError(t, err) { + httpClient := httpclient.GetHTTPClient() + req.SetBasicAuth(user.Username, defaultPassword) + resp, err := httpClient.Do(req) + if assert.NoError(t, err) { + defer resp.Body.Close() + assert.Equal(t, http.StatusOK, resp.StatusCode) + assert.Equal(t, "application/sftpgo", resp.Header.Get("Content-Type")) + } + } + + err = os.Remove(testFilePath) + assert.NoError(t, err) + _, err = httpdtest.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) +} + func TestHEAD(t *testing.T) { u := getTestUser() user, _, err := httpdtest.AddUser(u, http.StatusCreated) diff --git a/sftpgo.json b/sftpgo.json index dde5f911..64f85aca 100644 --- a/sftpgo.json +++ b/sftpgo.json @@ -181,7 +181,8 @@ }, "mime_types": { "enabled": true, - "max_size": 1000 + "max_size": 1000, + "custom_mappings": [] } } },