webclient: defer file list rendering

combined with server side processing I can now list a directory with
about 100.000 files in less than 2 seconds without losing client side
filtering and pagination

README.md

@@ -14,7 +14,7 @@ Several storage backends are supported: local filesystem, encrypted local filesy
 
 - Support for serving local filesystem, encrypted local filesystem, S3 Compatible Object Storage, Google Cloud Storage, Azure Blob Storage or other SFTP accounts over SFTP/SCP/FTP/WebDAV.
 - Virtual folders are supported: a virtual folder can use any of the supported storage backends. So you can have, for example, an S3 user that exposes a GCS bucket (or part of it) on a specified path and an encrypted local filesystem on another one. Virtual folders can be private or shared among multiple users, for shared virtual folders you can define different quota limits for each user.
-- Configurable custom commands and/or HTTP notifications on file upload, download, pre-delete, delete, rename, on SSH commands and on user add, update and delete.
+- Configurable custom commands and/or HTTP hooks on file upload, pre-upload, download, pre-download, delete, pre-delete, rename, on SSH commands and on user add, update and delete.
 - Virtual accounts stored within a "data provider".
 - SQLite, MySQL, PostgreSQL, CockroachDB, Bolt (key/value store in pure Go) and in-memory data providers are supported.
 - Chroot isolation for local accounts. Cloud-based accounts can be restricted to a certain base path.

dataprovider/admin.go

@@ -41,6 +41,7 @@ var (
 )
 
 // AdminFilters defines additional restrictions for SFTPGo admins
+// TODO: rename to AdminOptions in v3
 type AdminFilters struct {
 	// only clients connecting from these IP/Mask are allowed.
 	// IP/Mask must be in CIDR notation as defined in RFC 4632 and RFC 4291

httpd/httpd.go

@@ -74,6 +74,7 @@ const (
 	webTemplateFolderDefault        = "/web/admin/template/folder"
 	webClientLoginPathDefault       = "/web/client/login"
 	webClientFilesPathDefault       = "/web/client/files"
+	webClientDirContentsPathDefault = "/web/client/listdir"
 	webClientCredentialsPathDefault = "/web/client/credentials"
 	webChangeClientPwdPathDefault   = "/web/client/changepwd"
 	webChangeClientKeysPathDefault  = "/web/client/managekeys"
@@ -118,6 +119,7 @@ var (
 	webTemplateFolder        string
 	webClientLoginPath       string
 	webClientFilesPath       string
+	webClientDirContentsPath string
 	webClientCredentialsPath string
 	webChangeClientPwdPath   string
 	webChangeClientKeysPath  string
@@ -412,6 +414,7 @@ func updateWebClientURLs(baseURL string) {
 	webBaseClientPath = path.Join(baseURL, webBasePathClientDefault)
 	webClientLoginPath = path.Join(baseURL, webClientLoginPathDefault)
 	webClientFilesPath = path.Join(baseURL, webClientFilesPathDefault)
+	webClientDirContentsPath = path.Join(baseURL, webClientDirContentsPathDefault)
 	webClientCredentialsPath = path.Join(baseURL, webClientCredentialsPathDefault)
 	webChangeClientPwdPath = path.Join(baseURL, webChangeClientPwdPathDefault)
 	webChangeClientKeysPath = path.Join(baseURL, webChangeClientKeysPathDefault)

httpd/httpd_test.go

@@ -91,6 +91,7 @@ const (
 	webBasePathClient         = "/web/client"
 	webClientLoginPath        = "/web/client/login"
 	webClientFilesPath        = "/web/client/files"
+	webClientDirContentsPath  = "/web/client/listdir"
 	webClientCredentialsPath  = "/web/client/credentials"
 	webChangeClientPwdPath    = "/web/client/changepwd"
 	webChangeClientKeysPath   = "/web/client/managekeys"
@@ -4567,6 +4568,13 @@ func TestWebClientLoginMock(t *testing.T) {
 	setJWTCookieForReq(req, webToken)
 	rr = executeRequest(req)
 	checkResponseCode(t, http.StatusInternalServerError, rr)
+	assert.Contains(t, rr.Body.String(), "unable to retrieve your user")
+
+	req, _ = http.NewRequest(http.MethodGet, webClientDirContentsPath, nil)
+	setJWTCookieForReq(req, webToken)
+	rr = executeRequest(req)
+	checkResponseCode(t, http.StatusInternalServerError, rr)
+	assert.Contains(t, rr.Body.String(), "unable to retrieve your user")
 
 	csrfToken, err := getCSRFToken(httpBaseURL + webClientLoginPath)
 	assert.NoError(t, err)
@@ -4962,6 +4970,8 @@ func TestWebGetFiles(t *testing.T) {
 		err = os.WriteFile(filepath.Join(user.GetHomeDir(), testFileName+ext), testFileContents, os.ModePerm)
 		assert.NoError(t, err)
 	}
+	err = os.Symlink(filepath.Join(user.GetHomeDir(), testFileName+".doc"), filepath.Join(user.GetHomeDir(), testDir, testFileName+".link"))
+	assert.NoError(t, err)
 	webToken, err := getJWTWebClientTokenFromTestServer(defaultUsername, defaultPassword)
 	assert.NoError(t, err)
 	req, _ := http.NewRequest(http.MethodGet, webClientFilesPath, nil)
@@ -4974,6 +4984,29 @@ func TestWebGetFiles(t *testing.T) {
 	rr = executeRequest(req)
 	checkResponseCode(t, http.StatusOK, rr)
 
+	req, _ = http.NewRequest(http.MethodGet, webClientDirContentsPath+"?path="+testDir, nil)
+	setJWTCookieForReq(req, webToken)
+	rr = executeRequest(req)
+	checkResponseCode(t, http.StatusOK, rr)
+	var dirContents []map[string]string
+	err = json.Unmarshal(rr.Body.Bytes(), &dirContents)
+	assert.NoError(t, err)
+	assert.Len(t, dirContents, 1)
+
+	req, _ = http.NewRequest(http.MethodGet, webClientDirContentsPath+"?path=/", nil)
+	setJWTCookieForReq(req, webToken)
+	rr = executeRequest(req)
+	checkResponseCode(t, http.StatusOK, rr)
+	dirContents = nil
+	err = json.Unmarshal(rr.Body.Bytes(), &dirContents)
+	assert.NoError(t, err)
+	assert.Len(t, dirContents, len(extensions)+1)
+
+	req, _ = http.NewRequest(http.MethodGet, webClientDirContentsPath+"?path=/missing", nil)
+	setJWTCookieForReq(req, webToken)
+	rr = executeRequest(req)
+	checkResponseCode(t, http.StatusInternalServerError, rr)
+
 	req, _ = http.NewRequest(http.MethodGet, webClientFilesPath+"?path="+testFileName, nil)
 	setJWTCookieForReq(req, webToken)
 	rr = executeRequest(req)

httpd/internal_test.go

@@ -1352,6 +1352,13 @@ func TestGetFilesInvalidClaims(t *testing.T) {
 	handleClientGetFiles(rr, req)
 	assert.Equal(t, http.StatusForbidden, rr.Code)
 	assert.Contains(t, rr.Body.String(), "Invalid token claims")
+
+	rr = httptest.NewRecorder()
+	req, _ = http.NewRequest(http.MethodGet, webClientDirContentsPath, nil)
+	req.Header.Set("Cookie", fmt.Sprintf("jwt=%v", token["access_token"]))
+	handleClientGetDirContents(rr, req)
+	assert.Equal(t, http.StatusForbidden, rr.Code)
+	assert.Contains(t, rr.Body.String(), "invalid token claims")
 }
 
 func TestManageKeysInvalidClaims(t *testing.T) {

httpd/server.go

@@ -573,6 +573,7 @@ func (s *httpdServer) initializeRouter() {
 
 			router.Get(webClientLogoutPath, handleWebClientLogout)
 			router.With(s.refreshCookie).Get(webClientFilesPath, handleClientGetFiles)
+			router.With(compressor.Handler, s.refreshCookie).Get(webClientDirContentsPath, handleClientGetDirContents)
 			router.With(s.refreshCookie).Get(webClientCredentialsPath, handleClientGetCredentials)
 			router.Post(webChangeClientPwdPath, handleWebClientChangePwdPost)
 			router.With(checkClientPerm(dataprovider.WebClientPubKeyChangeDisabled)).

httpd/webclient.go

@@ -15,6 +15,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/go-chi/render"
 	"github.com/rs/xid"
 
 	"github.com/drakkan/sftpgo/common"
@@ -78,15 +79,11 @@ type dirMapping struct {
 
 type filesPage struct {
 	baseClientPage
-	CurrentDir          string
+	CurrentDir string
-	Files               []os.FileInfo
+	ReadDirURL string
-	Error               string
+	Files      []os.FileInfo
-	Paths               []dirMapping
+	Error      string
-	FormatTime          func(time.Time) string
+	Paths      []dirMapping
-	GetObjectURL        func(string, string) string
-	GetSize             func(int64) string
-	IsLink              func(os.FileInfo) bool
-	GetIconForExtension func(string) string
 }
 
 type clientMessagePage struct {
@@ -115,36 +112,6 @@ func getFileObjectModTime(t time.Time) string {
 	return t.Format("2006-01-02 15:04")
 }
 
-func isFileObjectLink(info os.FileInfo) bool {
-	return info.Mode()&os.ModeSymlink != 0
-}
-
-func getFileIconForExtension(name string) string {
-	switch path.Ext(name) {
-	case ".doc", ".docx", ".odt":
-		return "far fa-file-word"
-	case ".ppt", ".pptx":
-		return "far fa-file-powerpoint"
-	case ".xls", ".xlsx":
-		return "far fa-file-excel"
-	case ".pdf":
-		return "far fa-file-pdf"
-	case ".webm", ".mkv", ".flv", ".vob", ".ogv", ".ogg", ".avi", ".ts", ".mov", ".wmv", ".asf", ".mpeg", ".mpv", ".3gp":
-		return "far fa-file-video"
-	case ".jpeg", ".jpg", ".png", ".gif", ".webp", ".tiff", ".psd", ".bmp", ".svg", ".jp2":
-		return "far fa-file-image"
-	case ".go", ".java", ".php", ".cs", ".asp", ".aspx", ".css", ".html", ".js", ".py", ".rb", ".cgi", ".c", ".cpp", ".h",
-		".hpp", ".kt", ".ktm", ".kts", ".swift", ".r":
-		return "far fa-file-code"
-	case ".zip", ".rar", ".tar", ".gz", ".bz2", ".zstd", ".zst", ".sz", ".lz", ".lz4", ".xz":
-		return "far fa-file-archive"
-	case ".txt", ".sh", ".json", ".yaml", ".toml":
-		return "far fa-file-alt"
-	default:
-		return "far fa-file"
-	}
-}
-
 func loadClientTemplates(templatesPath string) {
 	filesPaths := []string{
 		filepath.Join(templatesPath, templateClientDir, templateClientBase),
@@ -248,15 +215,11 @@ func renderClientNotFoundPage(w http.ResponseWriter, r *http.Request, err error)
 
 func renderFilesPage(w http.ResponseWriter, r *http.Request, files []os.FileInfo, dirName, error string) {
 	data := filesPage{
-		baseClientPage:      getBaseClientPageData(pageClientFilesTitle, webClientFilesPath, r),
+		baseClientPage: getBaseClientPageData(pageClientFilesTitle, webClientFilesPath, r),
-		Files:               files,
+		Files:          files,
-		Error:               error,
+		Error:          error,
-		CurrentDir:          dirName,
+		CurrentDir:     url.QueryEscape(dirName),
-		FormatTime:          getFileObjectModTime,
+		ReadDirURL:     webClientDirContentsPath,
-		GetObjectURL:        getFileObjectURL,
-		GetSize:             utils.ByteCountIEC,
-		IsLink:              isFileObjectLink,
-		GetIconForExtension: getFileIconForExtension,
 	}
 	paths := []dirMapping{}
 	if dirName != "/" {
@@ -306,6 +269,60 @@ func handleWebClientLogout(w http.ResponseWriter, r *http.Request) {
 	http.Redirect(w, r, webClientLoginPath, http.StatusFound)
 }
 
+func handleClientGetDirContents(w http.ResponseWriter, r *http.Request) {
+	claims, err := getTokenClaims(r)
+	if err != nil || claims.Username == "" {
+		sendAPIResponse(w, r, nil, "invalid token claims", http.StatusForbidden)
+		return
+	}
+
+	user, err := dataprovider.UserExists(claims.Username)
+	if err != nil {
+		sendAPIResponse(w, r, nil, "unable to retrieve your user", http.StatusInternalServerError)
+		return
+	}
+
+	connection := &Connection{
+		BaseConnection: common.NewBaseConnection(xid.New().String(), common.ProtocolHTTP, user),
+		request:        r,
+	}
+	common.Connections.Add(connection)
+	defer common.Connections.Remove(connection.GetID())
+
+	name := "/"
+	if _, ok := r.URL.Query()["path"]; ok {
+		name = utils.CleanPath(r.URL.Query().Get("path"))
+	}
+
+	contents, err := connection.ReadDir(name)
+	if err != nil {
+		sendAPIResponse(w, r, nil, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	results := make([]map[string]string, 0, len(contents))
+	for _, info := range contents {
+		res := make(map[string]string)
+		if info.IsDir() {
+			res["type"] = "1"
+			res["size"] = ""
+		} else {
+			res["type"] = "2"
+			if info.Mode()&os.ModeSymlink != 0 {
+				res["size"] = ""
+			} else {
+				res["size"] = utils.ByteCountIEC(info.Size())
+			}
+		}
+		res["name"] = info.Name()
+		res["last_modified"] = getFileObjectModTime(info.ModTime())
+		res["url"] = getFileObjectURL(name, info.Name())
+		results = append(results, res)
+	}
+
+	render.JSON(w, r, results)
+}
+
 func handleClientGetFiles(w http.ResponseWriter, r *http.Request) {
 	claims, err := getTokenClaims(r)
 	if err != nil || claims.Username == "" {
@@ -315,7 +332,7 @@ func handleClientGetFiles(w http.ResponseWriter, r *http.Request) {
 
 	user, err := dataprovider.UserExists(claims.Username)
 	if err != nil {
-		renderClientInternalServerErrorPage(w, r, err)
+		renderClientInternalServerErrorPage(w, r, errors.New("unable to retrieve your user"))
 		return
 	}
 

templates/webadmin/connections.html

@@ -106,7 +106,7 @@
             },
             error: function ($xhr, textStatus, errorThrown) {
                 table.button('disconnect:name').enable(true);
-                var txt = "Unable to close the selected connection";
+                var txt = "Failed to close the selected connection";
                 if ($xhr) {
                     var json = $xhr.responseJSON;
                     if (json) {

templates/webclient/files.html

@@ -11,6 +11,9 @@
 {{end}}
 
 {{define "page_body"}}
+<div id="errorMsg" class="card mb-4 border-left-warning" style="display: none;">
+    <div id="errorTxt" class="card-body text-form-error"></div>
+</div>
 
 <div class="card shadow mb-4">
     <div class="card-header py-3">
@@ -32,24 +35,6 @@
                         <th>Last modified</th>
                     </tr>
                 </thead>
-                <tbody>
-                    {{range .Files}}
-                    {{if .IsDir}}
-                    <tr>
-                        <td>1</td>
-                        <td><i class="fas fa-folder"></i>&nbsp;<a href="{{call $.GetObjectURL $.CurrentDir .Name}}">{{.Name}}</a></td>
-                        <td></td>
-                        <td>{{call $.FormatTime .ModTime}}</td>
-                    {{else}}
-                    <tr>
-                        <td>2</td>
-                        <td><i class="{{if call $.IsLink .}}fas fa-external-link-alt{{else}}{{call $.GetIconForExtension .Name}}{{end}}"></i>&nbsp;<a href="{{call $.GetObjectURL $.CurrentDir .Name}}">{{.Name}}</a></td>
-                        <td>{{if not (call $.IsLink .)}}{{call $.GetSize .Size}}{{end}}</td>
-                        <td>{{call $.FormatTime .ModTime}}</td>
-                    </tr>
-                    {{end}}
-                    {{end}}
-                </tbody>
             </table>
         </div>
     </div>
@@ -66,6 +51,96 @@
 <script src="{{.StaticURL}}/vendor/datatables/responsive.bootstrap4.min.js"></script>
 <script src="{{.StaticURL}}/vendor/datatables/dataTables.select.min.js"></script>
 <script type="text/javascript">
+
+    function getIconForFile(filename) {
+            var extension = filename.slice((filename.lastIndexOf(".") - 1 >>> 0) + 2).toLowerCase();
+
+            switch (extension) {
+                case "doc":
+                case "docx":
+                case "odt":
+                    return "far fa-file-word";
+                case "ppt":
+                case "pptx":
+                    return "far fa-file-powerpoint";
+                case "xls":
+                case "xlsx":
+                case "ods":
+                    return "far fa-file-excel";
+                case "pdf":
+                    return "far fa-file-pdf";
+                case "webm":
+                case "mkv":
+                case "flv":
+                case "vob":
+                case "ogv":
+                case "ogg":
+                case "avi":
+                case "ts":
+                case "mov":
+                case "wmv":
+                case "asf":
+                case "mpeg":
+                case "mpv":
+                case "3gp":
+                    return "far fa-file-video";
+                case "jpeg":
+                case "jpg":
+                case "png":
+                case "gif":
+                case "webp":
+                case "tiff":
+                case "psd":
+                case "bmp":
+                case "svg":
+                case "jp2":
+                    return "far fa-file-image";
+                case "go":
+                case "sh":
+                case "java":
+                case "php":
+                case "cs":
+                case "asp":
+                case "aspx":
+                case "css":
+                case "html":
+                case "js":
+                case "py":
+                case "rb":
+                case "cgi":
+                case "c":
+                case "cpp":
+                case "h":
+                case "hpp":
+                case "kt":
+                case "ktm":
+                case "kts":
+                case "swift":
+                case "r":
+                    return "far fa-file-code";
+                case "zip":
+                case "rar":
+                case "tar":
+                case "gz":
+                case "bz2":
+                case "zstd":
+                case "zst":
+                case "sz":
+                case "lz":
+                case "lz4":
+                case "xz":
+                    return "far fa-file-archive";
+                case "txt":
+                case "json":
+                case "yaml":
+                case "toml":
+                    return "far fa-file-alt";
+                default:
+                    return "far fa-file";
+            }
+        }
+
+
     $(document).ready(function () {
         $.fn.dataTable.ext.buttons.refresh = {
             text: '<i class="fas fa-sync-alt"></i>',
@@ -77,6 +152,50 @@
         };
 
         var table = $('#dataTable').DataTable({
+            "ajax": {
+                "url": "{{.ReadDirURL}}?path={{.CurrentDir}}",
+                "dataSrc": "",
+                "error": function ($xhr, textStatus, errorThrown) {
+                    $(".dataTables_processing").hide();
+                    var txt = "Failed to get directory listing";
+                    if ($xhr) {
+                        var json = $xhr.responseJSON;
+                        if (json) {
+                            txt += ": " + json.message;
+                        }
+                    }
+                    $('#errorTxt').text(txt);
+                    $('#errorMsg').show();
+                    setTimeout(function () {
+                        $('#errorMsg').hide();
+                    }, 10000);
+                }
+            },
+            "deferRender": true,
+            "processing": true,
+            "columns": [
+                { "data": "type" },
+                {
+                    "data": "name",
+                    "render": function(data, type, row){
+                        if (type === 'display') {
+                            if (row["type"] == "1"){
+                                return `<i class="fas fa-folder"></i>&nbsp;<a href="${row['url']}">${data}</a>`;
+                            } else {
+                                if (row["size"] == ""){
+                                    return `<i class="fas fa-external-link-alt"></i>&nbsp;<a href="${row['url']}">${data}</a>`;
+                                } else {
+                                    var icon = getIconForFile(data);
+                                    return `<i class="${icon}"></i>&nbsp;<a href="${row['url']}">${data}</a>`;
+                                }
+                            }
+                        }
+                        return data;
+                    }
+                },
+                { "data": "size" },
+                { "data": "last_modified" }
+            ],
             "buttons": [],
             "lengthChange": false,
             "columnDefs": [
@@ -94,21 +213,26 @@
             "scrollY": false,
             "responsive": true,
             "language": {
+                "processing": '<i class="fas fa-spinner fa-spin fa-3x fa-fw"></i><span class="sr-only">Loading...</span>',
+                "loadingRecords": "",
                 "emptyTable": "No files or folders"
             },
             /*"select": {
                 "style":    'single',
                 "blurable": true
             },*/
+            "initComplete": function(settings, json) {
+                table.button().add(0,'refresh');
+                table.button().add(0,'pageLength');
+                table.buttons().container().appendTo('#dataTable_wrapper .col-md-6:eq(0)');
+            },
             "orderFixed": [ 0, 'asc' ],
             "order": [[1, 'asc']]
         });
 
         new $.fn.dataTable.FixedHeader( table );
 
-        table.button().add(0,'refresh');
+        $.fn.dataTable.ext.errMode = 'none';
-        table.button().add(0,'pageLength');
-        table.buttons().container().appendTo('#dataTable_wrapper .col-md-6:eq(0)');
 
         /*table.on('select', function (e, dt, type, indexes) {
             if (type === 'row') {