webclient: defer file list rendering

combined with server side processing I can now list a directory with
about 100.000 files in less than 2 seconds without losing client side
filtering and pagination
This commit is contained in:
Nicola Murino 2021-05-27 09:40:46 +02:00
parent 25a44030f9
commit 7a85c66ee7
No known key found for this signature in database
GPG key ID: 2F1FB59433D5A8CB
9 changed files with 258 additions and 72 deletions

View file

@ -14,7 +14,7 @@ Several storage backends are supported: local filesystem, encrypted local filesy
- Support for serving local filesystem, encrypted local filesystem, S3 Compatible Object Storage, Google Cloud Storage, Azure Blob Storage or other SFTP accounts over SFTP/SCP/FTP/WebDAV.
- Virtual folders are supported: a virtual folder can use any of the supported storage backends. So you can have, for example, an S3 user that exposes a GCS bucket (or part of it) on a specified path and an encrypted local filesystem on another one. Virtual folders can be private or shared among multiple users, for shared virtual folders you can define different quota limits for each user.
- Configurable custom commands and/or HTTP notifications on file upload, download, pre-delete, delete, rename, on SSH commands and on user add, update and delete.
- Configurable custom commands and/or HTTP hooks on file upload, pre-upload, download, pre-download, delete, pre-delete, rename, on SSH commands and on user add, update and delete.
- Virtual accounts stored within a "data provider".
- SQLite, MySQL, PostgreSQL, CockroachDB, Bolt (key/value store in pure Go) and in-memory data providers are supported.
- Chroot isolation for local accounts. Cloud-based accounts can be restricted to a certain base path.

View file

@ -41,6 +41,7 @@ var (
)
// AdminFilters defines additional restrictions for SFTPGo admins
// TODO: rename to AdminOptions in v3
type AdminFilters struct {
// only clients connecting from these IP/Mask are allowed.
// IP/Mask must be in CIDR notation as defined in RFC 4632 and RFC 4291

View file

@ -74,6 +74,7 @@ const (
webTemplateFolderDefault = "/web/admin/template/folder"
webClientLoginPathDefault = "/web/client/login"
webClientFilesPathDefault = "/web/client/files"
webClientDirContentsPathDefault = "/web/client/listdir"
webClientCredentialsPathDefault = "/web/client/credentials"
webChangeClientPwdPathDefault = "/web/client/changepwd"
webChangeClientKeysPathDefault = "/web/client/managekeys"
@ -118,6 +119,7 @@ var (
webTemplateFolder string
webClientLoginPath string
webClientFilesPath string
webClientDirContentsPath string
webClientCredentialsPath string
webChangeClientPwdPath string
webChangeClientKeysPath string
@ -412,6 +414,7 @@ func updateWebClientURLs(baseURL string) {
webBaseClientPath = path.Join(baseURL, webBasePathClientDefault)
webClientLoginPath = path.Join(baseURL, webClientLoginPathDefault)
webClientFilesPath = path.Join(baseURL, webClientFilesPathDefault)
webClientDirContentsPath = path.Join(baseURL, webClientDirContentsPathDefault)
webClientCredentialsPath = path.Join(baseURL, webClientCredentialsPathDefault)
webChangeClientPwdPath = path.Join(baseURL, webChangeClientPwdPathDefault)
webChangeClientKeysPath = path.Join(baseURL, webChangeClientKeysPathDefault)

View file

@ -91,6 +91,7 @@ const (
webBasePathClient = "/web/client"
webClientLoginPath = "/web/client/login"
webClientFilesPath = "/web/client/files"
webClientDirContentsPath = "/web/client/listdir"
webClientCredentialsPath = "/web/client/credentials"
webChangeClientPwdPath = "/web/client/changepwd"
webChangeClientKeysPath = "/web/client/managekeys"
@ -4567,6 +4568,13 @@ func TestWebClientLoginMock(t *testing.T) {
setJWTCookieForReq(req, webToken)
rr = executeRequest(req)
checkResponseCode(t, http.StatusInternalServerError, rr)
assert.Contains(t, rr.Body.String(), "unable to retrieve your user")
req, _ = http.NewRequest(http.MethodGet, webClientDirContentsPath, nil)
setJWTCookieForReq(req, webToken)
rr = executeRequest(req)
checkResponseCode(t, http.StatusInternalServerError, rr)
assert.Contains(t, rr.Body.String(), "unable to retrieve your user")
csrfToken, err := getCSRFToken(httpBaseURL + webClientLoginPath)
assert.NoError(t, err)
@ -4962,6 +4970,8 @@ func TestWebGetFiles(t *testing.T) {
err = os.WriteFile(filepath.Join(user.GetHomeDir(), testFileName+ext), testFileContents, os.ModePerm)
assert.NoError(t, err)
}
err = os.Symlink(filepath.Join(user.GetHomeDir(), testFileName+".doc"), filepath.Join(user.GetHomeDir(), testDir, testFileName+".link"))
assert.NoError(t, err)
webToken, err := getJWTWebClientTokenFromTestServer(defaultUsername, defaultPassword)
assert.NoError(t, err)
req, _ := http.NewRequest(http.MethodGet, webClientFilesPath, nil)
@ -4974,6 +4984,29 @@ func TestWebGetFiles(t *testing.T) {
rr = executeRequest(req)
checkResponseCode(t, http.StatusOK, rr)
req, _ = http.NewRequest(http.MethodGet, webClientDirContentsPath+"?path="+testDir, nil)
setJWTCookieForReq(req, webToken)
rr = executeRequest(req)
checkResponseCode(t, http.StatusOK, rr)
var dirContents []map[string]string
err = json.Unmarshal(rr.Body.Bytes(), &dirContents)
assert.NoError(t, err)
assert.Len(t, dirContents, 1)
req, _ = http.NewRequest(http.MethodGet, webClientDirContentsPath+"?path=/", nil)
setJWTCookieForReq(req, webToken)
rr = executeRequest(req)
checkResponseCode(t, http.StatusOK, rr)
dirContents = nil
err = json.Unmarshal(rr.Body.Bytes(), &dirContents)
assert.NoError(t, err)
assert.Len(t, dirContents, len(extensions)+1)
req, _ = http.NewRequest(http.MethodGet, webClientDirContentsPath+"?path=/missing", nil)
setJWTCookieForReq(req, webToken)
rr = executeRequest(req)
checkResponseCode(t, http.StatusInternalServerError, rr)
req, _ = http.NewRequest(http.MethodGet, webClientFilesPath+"?path="+testFileName, nil)
setJWTCookieForReq(req, webToken)
rr = executeRequest(req)

View file

@ -1352,6 +1352,13 @@ func TestGetFilesInvalidClaims(t *testing.T) {
handleClientGetFiles(rr, req)
assert.Equal(t, http.StatusForbidden, rr.Code)
assert.Contains(t, rr.Body.String(), "Invalid token claims")
rr = httptest.NewRecorder()
req, _ = http.NewRequest(http.MethodGet, webClientDirContentsPath, nil)
req.Header.Set("Cookie", fmt.Sprintf("jwt=%v", token["access_token"]))
handleClientGetDirContents(rr, req)
assert.Equal(t, http.StatusForbidden, rr.Code)
assert.Contains(t, rr.Body.String(), "invalid token claims")
}
func TestManageKeysInvalidClaims(t *testing.T) {

View file

@ -573,6 +573,7 @@ func (s *httpdServer) initializeRouter() {
router.Get(webClientLogoutPath, handleWebClientLogout)
router.With(s.refreshCookie).Get(webClientFilesPath, handleClientGetFiles)
router.With(compressor.Handler, s.refreshCookie).Get(webClientDirContentsPath, handleClientGetDirContents)
router.With(s.refreshCookie).Get(webClientCredentialsPath, handleClientGetCredentials)
router.Post(webChangeClientPwdPath, handleWebClientChangePwdPost)
router.With(checkClientPerm(dataprovider.WebClientPubKeyChangeDisabled)).

View file

@ -15,6 +15,7 @@ import (
"strings"
"time"
"github.com/go-chi/render"
"github.com/rs/xid"
"github.com/drakkan/sftpgo/common"
@ -78,15 +79,11 @@ type dirMapping struct {
type filesPage struct {
baseClientPage
CurrentDir string
Files []os.FileInfo
Error string
Paths []dirMapping
FormatTime func(time.Time) string
GetObjectURL func(string, string) string
GetSize func(int64) string
IsLink func(os.FileInfo) bool
GetIconForExtension func(string) string
CurrentDir string
ReadDirURL string
Files []os.FileInfo
Error string
Paths []dirMapping
}
type clientMessagePage struct {
@ -115,36 +112,6 @@ func getFileObjectModTime(t time.Time) string {
return t.Format("2006-01-02 15:04")
}
func isFileObjectLink(info os.FileInfo) bool {
return info.Mode()&os.ModeSymlink != 0
}
func getFileIconForExtension(name string) string {
switch path.Ext(name) {
case ".doc", ".docx", ".odt":
return "far fa-file-word"
case ".ppt", ".pptx":
return "far fa-file-powerpoint"
case ".xls", ".xlsx":
return "far fa-file-excel"
case ".pdf":
return "far fa-file-pdf"
case ".webm", ".mkv", ".flv", ".vob", ".ogv", ".ogg", ".avi", ".ts", ".mov", ".wmv", ".asf", ".mpeg", ".mpv", ".3gp":
return "far fa-file-video"
case ".jpeg", ".jpg", ".png", ".gif", ".webp", ".tiff", ".psd", ".bmp", ".svg", ".jp2":
return "far fa-file-image"
case ".go", ".java", ".php", ".cs", ".asp", ".aspx", ".css", ".html", ".js", ".py", ".rb", ".cgi", ".c", ".cpp", ".h",
".hpp", ".kt", ".ktm", ".kts", ".swift", ".r":
return "far fa-file-code"
case ".zip", ".rar", ".tar", ".gz", ".bz2", ".zstd", ".zst", ".sz", ".lz", ".lz4", ".xz":
return "far fa-file-archive"
case ".txt", ".sh", ".json", ".yaml", ".toml":
return "far fa-file-alt"
default:
return "far fa-file"
}
}
func loadClientTemplates(templatesPath string) {
filesPaths := []string{
filepath.Join(templatesPath, templateClientDir, templateClientBase),
@ -248,15 +215,11 @@ func renderClientNotFoundPage(w http.ResponseWriter, r *http.Request, err error)
func renderFilesPage(w http.ResponseWriter, r *http.Request, files []os.FileInfo, dirName, error string) {
data := filesPage{
baseClientPage: getBaseClientPageData(pageClientFilesTitle, webClientFilesPath, r),
Files: files,
Error: error,
CurrentDir: dirName,
FormatTime: getFileObjectModTime,
GetObjectURL: getFileObjectURL,
GetSize: utils.ByteCountIEC,
IsLink: isFileObjectLink,
GetIconForExtension: getFileIconForExtension,
baseClientPage: getBaseClientPageData(pageClientFilesTitle, webClientFilesPath, r),
Files: files,
Error: error,
CurrentDir: url.QueryEscape(dirName),
ReadDirURL: webClientDirContentsPath,
}
paths := []dirMapping{}
if dirName != "/" {
@ -306,6 +269,60 @@ func handleWebClientLogout(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, webClientLoginPath, http.StatusFound)
}
func handleClientGetDirContents(w http.ResponseWriter, r *http.Request) {
claims, err := getTokenClaims(r)
if err != nil || claims.Username == "" {
sendAPIResponse(w, r, nil, "invalid token claims", http.StatusForbidden)
return
}
user, err := dataprovider.UserExists(claims.Username)
if err != nil {
sendAPIResponse(w, r, nil, "unable to retrieve your user", http.StatusInternalServerError)
return
}
connection := &Connection{
BaseConnection: common.NewBaseConnection(xid.New().String(), common.ProtocolHTTP, user),
request: r,
}
common.Connections.Add(connection)
defer common.Connections.Remove(connection.GetID())
name := "/"
if _, ok := r.URL.Query()["path"]; ok {
name = utils.CleanPath(r.URL.Query().Get("path"))
}
contents, err := connection.ReadDir(name)
if err != nil {
sendAPIResponse(w, r, nil, err.Error(), http.StatusInternalServerError)
return
}
results := make([]map[string]string, 0, len(contents))
for _, info := range contents {
res := make(map[string]string)
if info.IsDir() {
res["type"] = "1"
res["size"] = ""
} else {
res["type"] = "2"
if info.Mode()&os.ModeSymlink != 0 {
res["size"] = ""
} else {
res["size"] = utils.ByteCountIEC(info.Size())
}
}
res["name"] = info.Name()
res["last_modified"] = getFileObjectModTime(info.ModTime())
res["url"] = getFileObjectURL(name, info.Name())
results = append(results, res)
}
render.JSON(w, r, results)
}
func handleClientGetFiles(w http.ResponseWriter, r *http.Request) {
claims, err := getTokenClaims(r)
if err != nil || claims.Username == "" {
@ -315,7 +332,7 @@ func handleClientGetFiles(w http.ResponseWriter, r *http.Request) {
user, err := dataprovider.UserExists(claims.Username)
if err != nil {
renderClientInternalServerErrorPage(w, r, err)
renderClientInternalServerErrorPage(w, r, errors.New("unable to retrieve your user"))
return
}

View file

@ -106,7 +106,7 @@
},
error: function ($xhr, textStatus, errorThrown) {
table.button('disconnect:name').enable(true);
var txt = "Unable to close the selected connection";
var txt = "Failed to close the selected connection";
if ($xhr) {
var json = $xhr.responseJSON;
if (json) {

View file

@ -11,6 +11,9 @@
{{end}}
{{define "page_body"}}
<div id="errorMsg" class="card mb-4 border-left-warning" style="display: none;">
<div id="errorTxt" class="card-body text-form-error"></div>
</div>
<div class="card shadow mb-4">
<div class="card-header py-3">
@ -32,24 +35,6 @@
<th>Last modified</th>
</tr>
</thead>
<tbody>
{{range .Files}}
{{if .IsDir}}
<tr>
<td>1</td>
<td><i class="fas fa-folder"></i>&nbsp;<a href="{{call $.GetObjectURL $.CurrentDir .Name}}">{{.Name}}</a></td>
<td></td>
<td>{{call $.FormatTime .ModTime}}</td>
{{else}}
<tr>
<td>2</td>
<td><i class="{{if call $.IsLink .}}fas fa-external-link-alt{{else}}{{call $.GetIconForExtension .Name}}{{end}}"></i>&nbsp;<a href="{{call $.GetObjectURL $.CurrentDir .Name}}">{{.Name}}</a></td>
<td>{{if not (call $.IsLink .)}}{{call $.GetSize .Size}}{{end}}</td>
<td>{{call $.FormatTime .ModTime}}</td>
</tr>
{{end}}
{{end}}
</tbody>
</table>
</div>
</div>
@ -66,6 +51,96 @@
<script src="{{.StaticURL}}/vendor/datatables/responsive.bootstrap4.min.js"></script>
<script src="{{.StaticURL}}/vendor/datatables/dataTables.select.min.js"></script>
<script type="text/javascript">
function getIconForFile(filename) {
var extension = filename.slice((filename.lastIndexOf(".") - 1 >>> 0) + 2).toLowerCase();
switch (extension) {
case "doc":
case "docx":
case "odt":
return "far fa-file-word";
case "ppt":
case "pptx":
return "far fa-file-powerpoint";
case "xls":
case "xlsx":
case "ods":
return "far fa-file-excel";
case "pdf":
return "far fa-file-pdf";
case "webm":
case "mkv":
case "flv":
case "vob":
case "ogv":
case "ogg":
case "avi":
case "ts":
case "mov":
case "wmv":
case "asf":
case "mpeg":
case "mpv":
case "3gp":
return "far fa-file-video";
case "jpeg":
case "jpg":
case "png":
case "gif":
case "webp":
case "tiff":
case "psd":
case "bmp":
case "svg":
case "jp2":
return "far fa-file-image";
case "go":
case "sh":
case "java":
case "php":
case "cs":
case "asp":
case "aspx":
case "css":
case "html":
case "js":
case "py":
case "rb":
case "cgi":
case "c":
case "cpp":
case "h":
case "hpp":
case "kt":
case "ktm":
case "kts":
case "swift":
case "r":
return "far fa-file-code";
case "zip":
case "rar":
case "tar":
case "gz":
case "bz2":
case "zstd":
case "zst":
case "sz":
case "lz":
case "lz4":
case "xz":
return "far fa-file-archive";
case "txt":
case "json":
case "yaml":
case "toml":
return "far fa-file-alt";
default:
return "far fa-file";
}
}
$(document).ready(function () {
$.fn.dataTable.ext.buttons.refresh = {
text: '<i class="fas fa-sync-alt"></i>',
@ -77,6 +152,50 @@
};
var table = $('#dataTable').DataTable({
"ajax": {
"url": "{{.ReadDirURL}}?path={{.CurrentDir}}",
"dataSrc": "",
"error": function ($xhr, textStatus, errorThrown) {
$(".dataTables_processing").hide();
var txt = "Failed to get directory listing";
if ($xhr) {
var json = $xhr.responseJSON;
if (json) {
txt += ": " + json.message;
}
}
$('#errorTxt').text(txt);
$('#errorMsg').show();
setTimeout(function () {
$('#errorMsg').hide();
}, 10000);
}
},
"deferRender": true,
"processing": true,
"columns": [
{ "data": "type" },
{
"data": "name",
"render": function(data, type, row){
if (type === 'display') {
if (row["type"] == "1"){
return `<i class="fas fa-folder"></i>&nbsp;<a href="${row['url']}">${data}</a>`;
} else {
if (row["size"] == ""){
return `<i class="fas fa-external-link-alt"></i>&nbsp;<a href="${row['url']}">${data}</a>`;
} else {
var icon = getIconForFile(data);
return `<i class="${icon}"></i>&nbsp;<a href="${row['url']}">${data}</a>`;
}
}
}
return data;
}
},
{ "data": "size" },
{ "data": "last_modified" }
],
"buttons": [],
"lengthChange": false,
"columnDefs": [
@ -94,21 +213,26 @@
"scrollY": false,
"responsive": true,
"language": {
"processing": '<i class="fas fa-spinner fa-spin fa-3x fa-fw"></i><span class="sr-only">Loading...</span>',
"loadingRecords": "",
"emptyTable": "No files or folders"
},
/*"select": {
"style": 'single',
"blurable": true
},*/
"initComplete": function(settings, json) {
table.button().add(0,'refresh');
table.button().add(0,'pageLength');
table.buttons().container().appendTo('#dataTable_wrapper .col-md-6:eq(0)');
},
"orderFixed": [ 0, 'asc' ],
"order": [[1, 'asc']]
});
new $.fn.dataTable.FixedHeader( table );
table.button().add(0,'refresh');
table.button().add(0,'pageLength');
table.buttons().container().appendTo('#dataTable_wrapper .col-md-6:eq(0)');
$.fn.dataTable.ext.errMode = 'none';
/*table.on('select', function (e, dt, type, indexes) {
if (type === 'row') {