WebDAV: allow to define custom MIME type mappings

Fixes #1154

Signed-off-by: Nicola Murino <nicola.murino@gmail.com>

docs/full-configuration.md

@@ -220,10 +220,16 @@ The configuration file contains the following sections:
     - `options_passthrough`, boolean.
     - `options_success_status`, integer.
     - `allow_private_network`, boolean.
-  - `cache` struct containing cache configuration for the authenticated users.
+  - `cache` struct containing cache configurations.
-    - `enabled`, boolean, set to true to enable user caching. Default: true.
+    - `users`, cache configuration for the authenticated users.
       - `expiration_time`, integer. Expiration time, in minutes, for the cached users. 0 means unlimited. Default: 0.
       - `max_size`, integer. Maximum number of users to cache. 0 means unlimited. Default: 50.
+    - `mime_types`, cache configuration for mime types.
+      - `enabled`, boolean, set to true to enable mime types caching. Default: `true`.
+      - `max_size`, integer. Maximum number of mime types to cache. 0 means no cache. Default: 1000.
+      - `custom_mappings`, additional mime types mapping. This is a platform independet way to add few additional mappings. You can set a limited number of mappings here, if you want to add a large list use the method provided by the OS of your choice. List of struct, each struct has the following fields:
+        - `ext`, string, file extension including the dot, for example `.json`
+        - `mime`, string, mime type, for example `application/json`
 
 </details>
 <details><summary><font size=4>Data Provider</font></summary>

go.mod

@@ -20,7 +20,7 @@ require (
 	github.com/bmatcuk/doublestar/v4 v4.6.0
 	github.com/cockroachdb/cockroach-go/v2 v2.2.20
 	github.com/coreos/go-oidc/v3 v3.5.0
-	github.com/drakkan/webdav v0.0.0-20221101181759-17ed21f9337b
+	github.com/drakkan/webdav v0.0.0-20230123134431-a95c027a0038
 	github.com/eikenb/pipeat v0.0.0-20210730190139-06b3e6902001
 	github.com/fclairamb/ftpserverlib v0.20.1-0.20230104020606-0b1a04eec221
 	github.com/fclairamb/go-log v0.4.1
@@ -104,7 +104,7 @@ require (
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
-	github.com/fatih/color v1.13.0 // indirect
+	github.com/fatih/color v1.14.1 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/go-jose/go-jose/v3 v3.0.0 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect

go.sum

@@ -851,8 +851,8 @@ github.com/drakkan/crypto v0.0.0-20230106095953-5417b4dfde62 h1:1Bk+GbTbF1PBu0id
 github.com/drakkan/crypto v0.0.0-20230106095953-5417b4dfde62/go.mod h1:eekSq7nI5pP2ZldL4867reOp0VL9TOfTaZa0DydSYk4=
 github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9 h1:LPH1dEblAOO/LoG7yHPMtBLXhQmjaga91/DDjWk9jWA=
 github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9/go.mod h1:2lmrmq866uF2tnje75wQHzmPXhmSWUt7Gyx2vgK1RCU=
-github.com/drakkan/webdav v0.0.0-20221101181759-17ed21f9337b h1:B9z7XyDoVxLO4yEvnXgdvZ+0Uw9NA1qdD4KTSGmKcoQ=
+github.com/drakkan/webdav v0.0.0-20230123134431-a95c027a0038 h1:vQe1F4uoOg7fmli8L/MyzbeAhf6SfM3M1bTGcgHscAw=
-github.com/drakkan/webdav v0.0.0-20221101181759-17ed21f9337b/go.mod h1:8opebuqUyBXrvl7Vo/S1Zzl9U0G1X2Ceud440eVuhUE=
+github.com/drakkan/webdav v0.0.0-20230123134431-a95c027a0038/go.mod h1:8opebuqUyBXrvl7Vo/S1Zzl9U0G1X2Ceud440eVuhUE=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
@@ -887,8 +887,9 @@ github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQL
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
 github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
-github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
+github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w=
+github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg=
 github.com/fclairamb/ftpserverlib v0.20.1-0.20230104020606-0b1a04eec221 h1:oIEBdcX1yNS5F+rk0xaDXMkwu9cT6+YSBEih45Wptec=
 github.com/fclairamb/ftpserverlib v0.20.1-0.20230104020606-0b1a04eec221/go.mod h1:2PS2QXGtruTtfUszbKGOuuWhDiK5u/GD9DK2DdAW+S8=
 github.com/fclairamb/go-log v0.4.1 h1:rLtdSG9x2pK41AIAnE8WYpl05xBJfw1ZyYxZaXFcBsM=

internal/config/config.go

@@ -311,6 +311,7 @@ func Init() {
 				MimeTypes: webdavd.MimeCacheConfig{
 					Enabled:        true,
 					MaxSize:        1000,
+					CustomMappings: nil,
 				},
 			},
 		},
@@ -732,6 +733,7 @@ func LoadConfig(configDir, configFile string) error {
 	}
 	// viper only supports slice of strings from env vars, so we use our custom method
 	loadBindingsFromEnv()
+	loadWebDAVCacheMappingsFromEnv()
 	resetInvalidConfigs()
 	logger.Debug(logSender, "", "config file used: '%#v', config loaded: %+v", viper.ConfigFileUsed(), getRedactedGlobalConf())
 	return nil
@@ -1249,6 +1251,27 @@ func getWebDAVDBindingProxyConfigsFromEnv(idx int, binding *webdavd.Binding) boo
 	return isSet
 }
 
+func loadWebDAVCacheMappingsFromEnv() []webdavd.CustomMimeMapping {
+	for idx := 0; idx < 30; idx++ {
+		ext, extOK := os.LookupEnv(fmt.Sprintf("SFTPGO_WEBDAVD__CACHE__MIME_TYPES__CUSTOM_MAPPINGS__%d__EXT", idx))
+		mime, mimeOK := os.LookupEnv(fmt.Sprintf("SFTPGO_WEBDAVD__CACHE__MIME_TYPES__CUSTOM_MAPPINGS__%d__MIME", idx))
+		if extOK && mimeOK {
+			if len(globalConf.WebDAVD.Cache.MimeTypes.CustomMappings) > idx {
+				globalConf.WebDAVD.Cache.MimeTypes.CustomMappings[idx].Ext = ext
+				globalConf.WebDAVD.Cache.MimeTypes.CustomMappings[idx].Mime = mime
+			} else {
+				globalConf.WebDAVD.Cache.MimeTypes.CustomMappings = append(globalConf.WebDAVD.Cache.MimeTypes.CustomMappings,
+					webdavd.CustomMimeMapping{
+						Ext:  ext,
+						Mime: mime,
+					})
+			}
+		}
+	}
+
+	return globalConf.WebDAVD.Cache.MimeTypes.CustomMappings
+}
+
 func getWebDAVDBindingFromEnv(idx int) {
 	binding := defaultWebDAVDBinding
 	if len(globalConf.WebDAVD.Bindings) > idx {
@@ -2010,6 +2033,7 @@ func setViperDefaults() {
 	viper.SetDefault("webdavd.cache.users.max_size", globalConf.WebDAVD.Cache.Users.MaxSize)
 	viper.SetDefault("webdavd.cache.mime_types.enabled", globalConf.WebDAVD.Cache.MimeTypes.Enabled)
 	viper.SetDefault("webdavd.cache.mime_types.max_size", globalConf.WebDAVD.Cache.MimeTypes.MaxSize)
+	viper.SetDefault("webdavd.cache.mime_types.custom_mappings", globalConf.WebDAVD.Cache.MimeTypes.CustomMappings)
 	viper.SetDefault("data_provider.driver", globalConf.ProviderConf.Driver)
 	viper.SetDefault("data_provider.name", globalConf.ProviderConf.Name)
 	viper.SetDefault("data_provider.host", globalConf.ProviderConf.Host)

internal/config/config_test.go

@@ -38,6 +38,7 @@ import (
 	"github.com/drakkan/sftpgo/v2/internal/sftpd"
 	"github.com/drakkan/sftpgo/v2/internal/smtp"
 	"github.com/drakkan/sftpgo/v2/internal/util"
+	"github.com/drakkan/sftpgo/v2/internal/webdavd"
 )
 
 const (
@@ -1031,6 +1032,72 @@ func TestFTPDBindingsFromEnv(t *testing.T) {
 	require.Equal(t, "cert.key", bindings[1].CertificateKeyFile)
 }
 
+func TestWebDAVMimeCache(t *testing.T) {
+	reset()
+
+	err := config.LoadConfig(configDir, "")
+	assert.NoError(t, err)
+	webdavdConf := config.GetWebDAVDConfig()
+	webdavdConf.Cache.MimeTypes.CustomMappings = []webdavd.CustomMimeMapping{
+		{
+			Ext:  ".custom",
+			Mime: "application/custom",
+		},
+	}
+	cfg := map[string]any{
+		"webdavd": webdavdConf,
+	}
+	data, err := json.Marshal(cfg)
+	assert.NoError(t, err)
+	confName := tempConfigName + ".json"
+	configFilePath := filepath.Join(configDir, confName)
+	err = os.WriteFile(configFilePath, data, 0666)
+	assert.NoError(t, err)
+
+	reset()
+	err = config.LoadConfig(configDir, confName)
+	assert.NoError(t, err)
+	mappings := config.GetWebDAVDConfig().Cache.MimeTypes.CustomMappings
+	if assert.Len(t, mappings, 1) {
+		assert.Equal(t, ".custom", mappings[0].Ext)
+		assert.Equal(t, "application/custom", mappings[0].Mime)
+	}
+	// now add from env
+	os.Setenv("SFTPGO_WEBDAVD__CACHE__MIME_TYPES__CUSTOM_MAPPINGS__1__EXT", ".custom1")
+	os.Setenv("SFTPGO_WEBDAVD__CACHE__MIME_TYPES__CUSTOM_MAPPINGS__1__MIME", "application/custom1")
+	t.Cleanup(func() {
+		os.Unsetenv("SFTPGO_WEBDAVD__CACHE__MIME_TYPES__CUSTOM_MAPPINGS__0__EXT")
+		os.Unsetenv("SFTPGO_WEBDAVD__CACHE__MIME_TYPES__CUSTOM_MAPPINGS__0__MIME")
+		os.Unsetenv("SFTPGO_WEBDAVD__CACHE__MIME_TYPES__CUSTOM_MAPPINGS__1__EXT")
+		os.Unsetenv("SFTPGO_WEBDAVD__CACHE__MIME_TYPES__CUSTOM_MAPPINGS__1__MIME")
+	})
+	reset()
+	err = config.LoadConfig(configDir, confName)
+	assert.NoError(t, err)
+	mappings = config.GetWebDAVDConfig().Cache.MimeTypes.CustomMappings
+	if assert.Len(t, mappings, 2) {
+		assert.Equal(t, ".custom", mappings[0].Ext)
+		assert.Equal(t, "application/custom", mappings[0].Mime)
+		assert.Equal(t, ".custom1", mappings[1].Ext)
+		assert.Equal(t, "application/custom1", mappings[1].Mime)
+	}
+	// override from env
+	os.Setenv("SFTPGO_WEBDAVD__CACHE__MIME_TYPES__CUSTOM_MAPPINGS__0__EXT", ".custom0")
+	os.Setenv("SFTPGO_WEBDAVD__CACHE__MIME_TYPES__CUSTOM_MAPPINGS__0__MIME", "application/custom0")
+	reset()
+	err = config.LoadConfig(configDir, confName)
+	assert.NoError(t, err)
+	mappings = config.GetWebDAVDConfig().Cache.MimeTypes.CustomMappings
+	if assert.Len(t, mappings, 2) {
+		assert.Equal(t, ".custom0", mappings[0].Ext)
+		assert.Equal(t, "application/custom0", mappings[0].Mime)
+		assert.Equal(t, ".custom1", mappings[1].Ext)
+		assert.Equal(t, "application/custom1", mappings[1].Mime)
+	}
+	err = os.Remove(configFilePath)
+	assert.NoError(t, err)
+}
+
 func TestWebDAVBindingsFromEnv(t *testing.T) {
 	reset()
 

internal/webdavd/file.go

@@ -84,6 +84,9 @@ type webDavFileInfo struct {
 // ContentType implements webdav.ContentTyper interface
 func (fi *webDavFileInfo) ContentType(ctx context.Context) (string, error) {
 	extension := path.Ext(fi.virtualPath)
+	if ctype, ok := customMimeTypeMapping[extension]; ok {
+		return ctype, nil
+	}
 	if extension == "" || extension == ".dat" {
 		return "application/octet-stream", nil
 	}

internal/webdavd/internal_test.go

@@ -712,6 +712,18 @@ func TestContentType(t *testing.T) {
 		assert.NoError(t, err)
 	}
 
+	baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath, testFilePath, testFile+".sftpgo",
+		common.TransferDownload, 0, 0, 0, 0, false, fs, dataprovider.TransferQuota{})
+	fs = newMockOsFs(false, fs.ConnectionID(), user.GetHomeDir(), nil, os.ErrInvalid)
+	davFile = newWebDavFile(baseTransfer, nil, nil)
+	davFile.Fs = fs
+	fi, err = davFile.Stat()
+	if assert.NoError(t, err) {
+		ctype, err := fi.(*webDavFileInfo).ContentType(ctx)
+		assert.NoError(t, err)
+		assert.Equal(t, "application/sftpgo", ctype)
+	}
+
 	baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath, testFilePath, testFile+".unknown2",
 		common.TransferDownload, 0, 0, 0, 0, false, fs, dataprovider.TransferQuota{})
 	fs = newMockOsFs(false, fs.ConnectionID(), user.GetHomeDir(), nil, os.ErrInvalid)

internal/webdavd/mimecache.go

@@ -22,7 +22,10 @@ type mimeCache struct {
 	mimeTypes map[string]string
 }
 
-var mimeTypeCache mimeCache
+var (
+	mimeTypeCache         mimeCache
+	customMimeTypeMapping map[string]string
+)
 
 func (c *mimeCache) addMimeToCache(key, value string) {
 	c.Lock()

internal/webdavd/webdavd.go

@@ -64,6 +64,12 @@ type CorsConfig struct {
 	AllowPrivateNetwork  bool     `json:"allow_private_network" mapstructure:"allow_private_network"`
 }
 
+// CustomMimeMapping defines additional, user defined mime mappings
+type CustomMimeMapping struct {
+	Ext  string `json:"ext" mapstructure:"ext"`
+	Mime string `json:"mime" mapstructure:"mime"`
+}
+
 // UsersCacheConfig defines the cache configuration for users
 type UsersCacheConfig struct {
 	ExpirationTime int `json:"expiration_time" mapstructure:"expiration_time"`
@@ -74,6 +80,7 @@ type UsersCacheConfig struct {
 type MimeCacheConfig struct {
 	Enabled        bool                `json:"enabled" mapstructure:"enabled"`
 	MaxSize        int                 `json:"max_size" mapstructure:"max_size"`
+	CustomMappings []CustomMimeMapping `json:"custom_mappings" mapstructure:"custom_mappings"`
 }
 
 // Cache configuration
@@ -228,7 +235,16 @@ func (c *Configuration) Initialize(configDir string) error {
 	}
 	if !c.Cache.MimeTypes.Enabled {
 		mimeTypeCache.maxSize = 0
+	} else {
+		customMimeTypeMapping = make(map[string]string)
+		for _, m := range c.Cache.MimeTypes.CustomMappings {
+			if m.Mime != "" {
+				logger.Debug(logSender, "", "adding custom mime mapping for extension %q, mime type %q", m.Ext, m.Mime)
+				customMimeTypeMapping[m.Ext] = m.Mime
 			}
+		}
+	}
+
 	if !c.ShouldBind() {
 		return common.ErrNoBinding
 	}

internal/webdavd/webdavd_test.go

@@ -272,6 +272,8 @@ func TestMain(m *testing.M) {
 	os.Setenv("SFTPGO_COMMON__ALLOW_SELF_CONNECTIONS", "1")
 	os.Setenv("SFTPGO_DEFAULT_ADMIN_USERNAME", "admin")
 	os.Setenv("SFTPGO_DEFAULT_ADMIN_PASSWORD", "password")
+	os.Setenv("SFTPGO_WEBDAVD__CACHE__MIME_TYPES__CUSTOM_MAPPINGS__0__EXT", ".sftpgo")
+	os.Setenv("SFTPGO_WEBDAVD__CACHE__MIME_TYPES__CUSTOM_MAPPINGS__0__MIME", "application/sftpgo")
 	err := config.LoadConfig(configDir, "")
 	if err != nil {
 		logger.ErrorToConsole("error loading configuration: %v", err)
@@ -2226,6 +2228,39 @@ func TestBytesRangeRequests(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+func TestContentTypeGET(t *testing.T) {
+	user, _, err := httpdtest.AddUser(getTestUser(), http.StatusCreated)
+	assert.NoError(t, err)
+
+	testFilePath := filepath.Join(homeBasePath, testFileName)
+	testFileSize := int64(64)
+	err = createTestFile(testFilePath, testFileSize)
+	assert.NoError(t, err)
+	client := getWebDavClient(user, true, nil)
+	err = uploadFileWithRawClient(testFilePath, testFileName+".sftpgo", user.Username, defaultPassword,
+		true, testFileSize, client)
+	assert.NoError(t, err)
+	remotePath := fmt.Sprintf("http://%v/%v", webDavServerAddr, testFileName+".sftpgo")
+	req, err := http.NewRequest(http.MethodGet, remotePath, nil)
+	if assert.NoError(t, err) {
+		httpClient := httpclient.GetHTTPClient()
+		req.SetBasicAuth(user.Username, defaultPassword)
+		resp, err := httpClient.Do(req)
+		if assert.NoError(t, err) {
+			defer resp.Body.Close()
+			assert.Equal(t, http.StatusOK, resp.StatusCode)
+			assert.Equal(t, "application/sftpgo", resp.Header.Get("Content-Type"))
+		}
+	}
+
+	err = os.Remove(testFilePath)
+	assert.NoError(t, err)
+	_, err = httpdtest.RemoveUser(user, http.StatusOK)
+	assert.NoError(t, err)
+	err = os.RemoveAll(user.GetHomeDir())
+	assert.NoError(t, err)
+}
+
 func TestHEAD(t *testing.T) {
 	u := getTestUser()
 	user, _, err := httpdtest.AddUser(u, http.StatusCreated)

sftpgo.json

@@ -181,7 +181,8 @@
       },
       "mime_types": {
         "enabled": true,
-        "max_size": 1000
+        "max_size": 1000,
+        "custom_mappings": []
       }
     }
   },