From e5836c8118b03936025dadf613b706663156b1e6 Mon Sep 17 00:00:00 2001 From: Nicola Murino Date: Sun, 4 Feb 2024 18:16:10 +0100 Subject: [PATCH] WebUI: add a JSON helper function Signed-off-by: Nicola Murino --- go.mod | 2 +- go.sum | 4 +- internal/httpd/api_utils.go | 35 ++++++++ internal/httpd/httpd_test.go | 137 +++++++++++++++++++++++--------- internal/httpd/internal_test.go | 36 +++++++++ internal/httpd/webadmin.go | 127 +++++++++++++---------------- internal/httpd/webclient.go | 18 ++--- 7 files changed, 239 insertions(+), 120 deletions(-) diff --git a/go.mod b/go.mod index 65e49e02..ba74673b 100644 --- a/go.mod +++ b/go.mod @@ -52,7 +52,7 @@ require ( github.com/robfig/cron/v3 v3.0.1 github.com/rs/cors v1.10.1 github.com/rs/xid v1.5.0 - github.com/rs/zerolog v1.31.0 + github.com/rs/zerolog v1.32.0 github.com/sftpgo/sdk v0.1.6-0.20240114195211-3f4916cc829c github.com/shirou/gopsutil/v3 v3.24.1 github.com/spf13/afero v1.11.0 diff --git a/go.sum b/go.sum index 1b8e9aac..8494987d 100644 --- a/go.sum +++ b/go.sum @@ -338,8 +338,8 @@ github.com/rs/cors v1.10.1 h1:L0uuZVXIKlI1SShY2nhFfo44TYvDPQ1w4oFkUJNfhyo= github.com/rs/cors v1.10.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A= -github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= +github.com/rs/zerolog v1.32.0 h1:keLypqrlIjaFsbmJOBdB/qvyF8KEtCWHwobLp5l/mQ0= +github.com/rs/zerolog v1.32.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= diff --git a/internal/httpd/api_utils.go b/internal/httpd/api_utils.go index dd31149a..b317dcf6 100644 --- a/internal/httpd/api_utils.go +++ b/internal/httpd/api_utils.go @@ -299,6 +299,41 @@ func renderAPIDirContents(w http.ResponseWriter, r *http.Request, contents []os. render.JSON(w, r, results) } +func streamData(w io.Writer, data []byte) { + b := bytes.NewBuffer(data) + _, err := io.CopyN(w, b, int64(len(data))) + if err != nil { + panic(http.ErrAbortHandler) + } +} + +func streamJSONArray(w http.ResponseWriter, chunkSize int, dataGetter func(limit, offset int) ([]byte, int, error)) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Accept-Ranges", "none") + w.WriteHeader(http.StatusOK) + + streamData(w, []byte("[")) + offset := 0 + for { + data, count, err := dataGetter(chunkSize, offset) + if err != nil { + panic(http.ErrAbortHandler) + } + if count == 0 { + break + } + if offset > 0 { + streamData(w, []byte(",")) + } + streamData(w, data[1:len(data)-1]) + if count < chunkSize { + break + } + offset += count + } + streamData(w, []byte("]")) +} + func getCompressedFileName(username string, files []string) string { if len(files) == 1 { name := path.Base(files[0]) diff --git a/internal/httpd/httpd_test.go b/internal/httpd/httpd_test.go index 2f823c4a..051e0a8d 100644 --- a/internal/httpd/httpd_test.go +++ b/internal/httpd/httpd_test.go @@ -7014,11 +7014,18 @@ func TestProviderErrors(t *testing.T) { assert.Equal(t, http.StatusOK, rr.Code) assert.Contains(t, rr.Body.String(), util.I18nErrorGetUser) - req, err = http.NewRequest(http.MethodGet, webClientSharesPath+jsonAPISuffix, nil) - assert.NoError(t, err) - setJWTCookieForReq(req, userWebToken) - rr = executeRequest(req) - checkResponseCode(t, http.StatusInternalServerError, rr) + getJSONShares := func() { + defer func() { + rcv := recover() + assert.Equal(t, http.ErrAbortHandler, rcv) + }() + req, err := http.NewRequest(http.MethodGet, webClientSharesPath+jsonAPISuffix, nil) + assert.NoError(t, err) + setJWTCookieForReq(req, userWebToken) + executeRequest(req) + } + getJSONShares() + req, err = http.NewRequest(http.MethodGet, webClientSharePath, nil) assert.NoError(t, err) setJWTCookieForReq(req, userWebToken) @@ -7256,11 +7263,19 @@ func TestProviderErrors(t *testing.T) { setJWTCookieForReq(req, testServerToken) rr = executeRequest(req) checkResponseCode(t, http.StatusInternalServerError, rr) - req, err = http.NewRequest(http.MethodGet, webAdminEventActionsPath+jsonAPISuffix, nil) - assert.NoError(t, err) - setJWTCookieForReq(req, testServerToken) - rr = executeRequest(req) - checkResponseCode(t, http.StatusInternalServerError, rr) + + getJSONActions := func() { + defer func() { + rcv := recover() + assert.Equal(t, http.ErrAbortHandler, rcv) + }() + req, err := http.NewRequest(http.MethodGet, webAdminEventActionsPath+jsonAPISuffix, nil) + assert.NoError(t, err) + setJWTCookieForReq(req, testServerToken) + executeRequest(req) + } + getJSONActions() + req, err = http.NewRequest(http.MethodGet, path.Join(webAdminEventRulePath, "rulename"), nil) assert.NoError(t, err) setJWTCookieForReq(req, testServerToken) @@ -7271,11 +7286,19 @@ func TestProviderErrors(t *testing.T) { setJWTCookieForReq(req, testServerToken) rr = executeRequest(req) checkResponseCode(t, http.StatusInternalServerError, rr) - req, err = http.NewRequest(http.MethodGet, webAdminEventRulesPath+jsonAPISuffix+"?qlimit=10", nil) - assert.NoError(t, err) - setJWTCookieForReq(req, testServerToken) - rr = executeRequest(req) - checkResponseCode(t, http.StatusInternalServerError, rr) + + getJSONRules := func() { + defer func() { + rcv := recover() + assert.Equal(t, http.ErrAbortHandler, rcv) + }() + req, err := http.NewRequest(http.MethodGet, webAdminEventRulesPath+jsonAPISuffix, nil) + assert.NoError(t, err) + setJWTCookieForReq(req, testServerToken) + executeRequest(req) + } + getJSONRules() + req, err = http.NewRequest(http.MethodGet, webAdminEventRulePath, nil) assert.NoError(t, err) setJWTCookieForReq(req, testServerToken) @@ -18525,7 +18548,7 @@ func TestWebUserShare(t *testing.T) { rr = executeRequest(req) checkResponseCode(t, http.StatusOK, rr) - req, err = http.NewRequest(http.MethodGet, webClientSharesPath+jsonAPISuffix, nil) //nolint:goconst + req, err = http.NewRequest(http.MethodGet, webClientSharesPath+jsonAPISuffix, nil) assert.NoError(t, err) req.RemoteAddr = defaultRemoteAddr setJWTCookieForReq(req, token) @@ -22878,6 +22901,13 @@ func TestWebEventAction(t *testing.T) { setCSRFHeaderForReq(req, csrfToken) rr = executeRequest(req) checkResponseCode(t, http.StatusOK, rr) + + req, err = http.NewRequest(http.MethodGet, webAdminEventActionsPath+jsonAPISuffix, nil) + assert.NoError(t, err) + setJWTCookieForReq(req, webToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusOK, rr) + assert.Equal(t, `[]`, rr.Body.String()) } func TestWebEventRule(t *testing.T) { @@ -24928,18 +24958,39 @@ func TestProviderClosedMock(t *testing.T) { rr = executeRequest(req) checkResponseCode(t, http.StatusInternalServerError, rr) - req, _ = http.NewRequest(http.MethodGet, webFoldersPath+jsonAPISuffix, nil) - setJWTCookieForReq(req, token) - rr = executeRequest(req) - checkResponseCode(t, http.StatusInternalServerError, rr) - req, _ = http.NewRequest(http.MethodGet, webGroupsPath+jsonAPISuffix, nil) - setJWTCookieForReq(req, token) - rr = executeRequest(req) - checkResponseCode(t, http.StatusInternalServerError, rr) - req, _ = http.NewRequest(http.MethodGet, webUsersPath+jsonAPISuffix, nil) - setJWTCookieForReq(req, token) - rr = executeRequest(req) - checkResponseCode(t, http.StatusInternalServerError, rr) + getJSONFolders := func() { + defer func() { + rcv := recover() + assert.Equal(t, http.ErrAbortHandler, rcv) + }() + req, _ := http.NewRequest(http.MethodGet, webFoldersPath+jsonAPISuffix, nil) + setJWTCookieForReq(req, token) + executeRequest(req) + } + getJSONFolders() + + getJSONGroups := func() { + defer func() { + rcv := recover() + assert.Equal(t, http.ErrAbortHandler, rcv) + }() + req, _ := http.NewRequest(http.MethodGet, webGroupsPath+jsonAPISuffix, nil) + setJWTCookieForReq(req, token) + executeRequest(req) + } + getJSONGroups() + + getJSONUsers := func() { + defer func() { + rcv := recover() + assert.Equal(t, http.ErrAbortHandler, rcv) + }() + req, _ := http.NewRequest(http.MethodGet, webUsersPath+jsonAPISuffix, nil) + setJWTCookieForReq(req, token) + executeRequest(req) + } + getJSONUsers() + req, _ = http.NewRequest(http.MethodGet, webUserPath+"/0", nil) setJWTCookieForReq(req, token) rr = executeRequest(req) @@ -24961,10 +25012,16 @@ func TestProviderClosedMock(t *testing.T) { rr = executeRequest(req) checkResponseCode(t, http.StatusInternalServerError, rr) - req, _ = http.NewRequest(http.MethodGet, webAdminsPath+jsonAPISuffix, nil) - setJWTCookieForReq(req, token) - rr = executeRequest(req) - checkResponseCode(t, http.StatusInternalServerError, rr) + getJSONAdmins := func() { + defer func() { + rcv := recover() + assert.Equal(t, http.ErrAbortHandler, rcv) + }() + req, _ := http.NewRequest(http.MethodGet, webAdminsPath+jsonAPISuffix, nil) + setJWTCookieForReq(req, token) + executeRequest(req) + } + getJSONAdmins() req, _ = http.NewRequest(http.MethodGet, path.Join(webFolderPath, defaultTokenAuthUser), nil) setJWTCookieForReq(req, token) @@ -24998,11 +25055,17 @@ func TestProviderClosedMock(t *testing.T) { rr = executeRequest(req) checkResponseCode(t, http.StatusInternalServerError, rr) - req, err = http.NewRequest(http.MethodGet, webAdminRolesPath+jsonAPISuffix, nil) - assert.NoError(t, err) - setJWTCookieForReq(req, token) - rr = executeRequest(req) - checkResponseCode(t, http.StatusInternalServerError, rr) + getJSONRoles := func() { + defer func() { + rcv := recover() + assert.Equal(t, http.ErrAbortHandler, rcv) + }() + req, err := http.NewRequest(http.MethodGet, webAdminRolesPath+jsonAPISuffix, nil) + assert.NoError(t, err) + setJWTCookieForReq(req, token) + executeRequest(req) + } + getJSONRoles() req, err = http.NewRequest(http.MethodGet, path.Join(webAdminRolePath, role.Name), nil) assert.NoError(t, err) diff --git a/internal/httpd/internal_test.go b/internal/httpd/internal_test.go index 9db01c4b..095adbfd 100644 --- a/internal/httpd/internal_test.go +++ b/internal/httpd/internal_test.go @@ -2195,6 +2195,33 @@ func TestRecoverer(t *testing.T) { assert.Equal(t, http.StatusInternalServerError, rr.Code, rr.Body.String()) } +func TestStreamJSONArray(t *testing.T) { + dataGetter := func(limit, offset int) ([]byte, int, error) { + return nil, 0, nil + } + rr := httptest.NewRecorder() + streamJSONArray(rr, 10, dataGetter) + assert.Equal(t, `[]`, rr.Body.String()) + + data := []int{} + for i := 0; i < 10; i++ { + data = append(data, i) + } + + dataGetter = func(limit, offset int) ([]byte, int, error) { + if offset >= len(data) { + return nil, 0, nil + } + val := data[offset] + data, err := json.Marshal([]int{val}) + return data, 1, err + } + + rr = httptest.NewRecorder() + streamJSONArray(rr, 1, dataGetter) + assert.Equal(t, `[0,1,2,3,4,5,6,7,8,9]`, rr.Body.String()) +} + func TestCompressorAbortHandler(t *testing.T) { defer func() { rcv := recover() @@ -2209,6 +2236,15 @@ func TestCompressorAbortHandler(t *testing.T) { renderCompressedFiles(&failingWriter{}, connection, "", nil, share) } +func TestStreamDataAbortHandler(t *testing.T) { + defer func() { + rcv := recover() + assert.Equal(t, http.ErrAbortHandler, rcv) + }() + + streamData(&failingWriter{}, []byte(`["a":"b"]`)) +} + func TestZipErrors(t *testing.T) { user := dataprovider.User{ BaseUser: sdk.BaseUser{ diff --git a/internal/httpd/webadmin.go b/internal/httpd/webadmin.go index a02f7998..639ca98c 100644 --- a/internal/httpd/webadmin.go +++ b/internal/httpd/webadmin.go @@ -16,6 +16,7 @@ package httpd import ( "context" + "encoding/json" "errors" "fmt" "html/template" @@ -2878,19 +2879,17 @@ func getAllAdmins(w http.ResponseWriter, r *http.Request) { sendAPIResponse(w, r, nil, util.I18nErrorInvalidToken, http.StatusForbidden) return } - admins := make([]dataprovider.Admin, 0, 50) - for { - a, err := dataprovider.GetAdmins(defaultQueryLimit, len(admins), dataprovider.OrderASC) + + dataGetter := func(limit, offset int) ([]byte, int, error) { + results, err := dataprovider.GetAdmins(limit, offset, dataprovider.OrderASC) if err != nil { - sendAPIResponse(w, r, err, getI18NErrorString(err, util.I18nError500Message), http.StatusInternalServerError) - return - } - admins = append(admins, a...) - if len(a) < defaultQueryLimit { - break + return nil, 0, err } + data, err := json.Marshal(results) + return data, len(results), err } - render.JSON(w, r, admins) + + streamJSONArray(w, defaultQueryLimit, dataGetter) } func (s *httpdServer) handleGetWebAdmins(w http.ResponseWriter, r *http.Request) { @@ -3043,19 +3042,17 @@ func getAllUsers(w http.ResponseWriter, r *http.Request) { sendAPIResponse(w, r, nil, util.I18nErrorInvalidToken, http.StatusForbidden) return } - users := make([]dataprovider.User, 0, 100) - for { - u, err := dataprovider.GetUsers(defaultQueryLimit, len(users), dataprovider.OrderASC, claims.Role) + + dataGetter := func(limit, offset int) ([]byte, int, error) { + results, err := dataprovider.GetUsers(limit, offset, dataprovider.OrderASC, claims.Role) if err != nil { - sendAPIResponse(w, r, err, getI18NErrorString(err, util.I18nError500Message), http.StatusInternalServerError) - return - } - users = append(users, u...) - if len(u) < defaultQueryLimit { - break + return nil, 0, err } + data, err := json.Marshal(results) + return data, len(results), err } - render.JSON(w, r, users) + + streamJSONArray(w, defaultQueryLimit, dataGetter) } func (s *httpdServer) handleGetWebUsers(w http.ResponseWriter, r *http.Request) { @@ -3538,19 +3535,17 @@ func (s *httpdServer) getWebVirtualFolders(w http.ResponseWriter, r *http.Reques func getAllFolders(w http.ResponseWriter, r *http.Request) { r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize) - folders := make([]vfs.BaseVirtualFolder, 0, 50) - for { - f, err := dataprovider.GetFolders(defaultQueryLimit, len(folders), dataprovider.OrderASC, false) + + dataGetter := func(limit, offset int) ([]byte, int, error) { + results, err := dataprovider.GetFolders(limit, offset, dataprovider.OrderASC, false) if err != nil { - sendAPIResponse(w, r, err, getI18NErrorString(err, util.I18nError500Message), http.StatusInternalServerError) - return - } - folders = append(folders, f...) - if len(f) < defaultQueryLimit { - break + return nil, 0, err } + data, err := json.Marshal(results) + return data, len(results), err } - render.JSON(w, r, folders) + + streamJSONArray(w, defaultQueryLimit, dataGetter) } func (s *httpdServer) handleWebGetFolders(w http.ResponseWriter, r *http.Request) { @@ -3578,19 +3573,17 @@ func (s *httpdServer) getWebGroups(w http.ResponseWriter, r *http.Request, limit func getAllGroups(w http.ResponseWriter, r *http.Request) { r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize) - groups := make([]dataprovider.Group, 0, 50) - for { - f, err := dataprovider.GetGroups(defaultQueryLimit, len(groups), dataprovider.OrderASC, false) + + dataGetter := func(limit, offset int) ([]byte, int, error) { + results, err := dataprovider.GetGroups(limit, offset, dataprovider.OrderASC, false) if err != nil { - sendAPIResponse(w, r, err, getI18NErrorString(err, util.I18nError500Message), http.StatusInternalServerError) - return - } - groups = append(groups, f...) - if len(f) < defaultQueryLimit { - break + return nil, 0, err } + data, err := json.Marshal(results) + return data, len(results), err } - render.JSON(w, r, groups) + + streamJSONArray(w, defaultQueryLimit, dataGetter) } func (s *httpdServer) handleWebGetGroups(w http.ResponseWriter, r *http.Request) { @@ -3707,19 +3700,17 @@ func (s *httpdServer) getWebEventActions(w http.ResponseWriter, r *http.Request, func getAllActions(w http.ResponseWriter, r *http.Request) { r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize) - actions := make([]dataprovider.BaseEventAction, 0, 10) - for { - res, err := dataprovider.GetEventActions(defaultQueryLimit, len(actions), dataprovider.OrderASC, false) + + dataGetter := func(limit, offset int) ([]byte, int, error) { + results, err := dataprovider.GetEventActions(limit, offset, dataprovider.OrderASC, false) if err != nil { - sendAPIResponse(w, r, err, getI18NErrorString(err, util.I18nError500Message), http.StatusInternalServerError) - return - } - actions = append(actions, res...) - if len(res) < defaultQueryLimit { - break + return nil, 0, err } + data, err := json.Marshal(results) + return data, len(results), err } - render.JSON(w, r, actions) + + streamJSONArray(w, defaultQueryLimit, dataGetter) } func (s *httpdServer) handleWebGetEventActions(w http.ResponseWriter, r *http.Request) { @@ -3819,19 +3810,17 @@ func (s *httpdServer) handleWebUpdateEventActionPost(w http.ResponseWriter, r *h func getAllRules(w http.ResponseWriter, r *http.Request) { r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize) - rules := make([]dataprovider.EventRule, 0, 10) - for { - res, err := dataprovider.GetEventRules(defaultQueryLimit, len(rules), dataprovider.OrderASC) + + dataGetter := func(limit, offset int) ([]byte, int, error) { + results, err := dataprovider.GetEventRules(limit, offset, dataprovider.OrderASC) if err != nil { - sendAPIResponse(w, r, err, getI18NErrorString(err, util.I18nError500Message), http.StatusInternalServerError) - return - } - rules = append(rules, res...) - if len(res) < defaultQueryLimit { - break + return nil, 0, err } + data, err := json.Marshal(results) + return data, len(results), err } - render.JSON(w, r, rules) + + streamJSONArray(w, defaultQueryLimit, dataGetter) } func (s *httpdServer) handleWebGetEventRules(w http.ResponseWriter, r *http.Request) { @@ -3942,19 +3931,17 @@ func (s *httpdServer) getWebRoles(w http.ResponseWriter, r *http.Request, limit func getAllRoles(w http.ResponseWriter, r *http.Request) { r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize) - roles := make([]dataprovider.Role, 0, 10) - for { - res, err := dataprovider.GetRoles(defaultQueryLimit, len(roles), dataprovider.OrderASC, false) + + dataGetter := func(limit, offset int) ([]byte, int, error) { + results, err := dataprovider.GetRoles(limit, offset, dataprovider.OrderASC, false) if err != nil { - sendAPIResponse(w, r, err, getI18NErrorString(err, util.I18nError500Message), http.StatusInternalServerError) - return - } - roles = append(roles, res...) - if len(res) < defaultQueryLimit { - break + return nil, 0, err } + data, err := json.Marshal(results) + return data, len(results), err } - render.JSON(w, r, roles) + + streamJSONArray(w, defaultQueryLimit, dataGetter) } func (s *httpdServer) handleWebGetRoles(w http.ResponseWriter, r *http.Request) { diff --git a/internal/httpd/webclient.go b/internal/httpd/webclient.go index 1d0fb3f7..e8a5f5bd 100644 --- a/internal/httpd/webclient.go +++ b/internal/httpd/webclient.go @@ -1521,19 +1521,17 @@ func getAllShares(w http.ResponseWriter, r *http.Request) { sendAPIResponse(w, r, nil, util.I18nErrorInvalidToken, http.StatusForbidden) return } - shares := make([]dataprovider.Share, 0, 10) - for { - sh, err := dataprovider.GetShares(defaultQueryLimit, len(shares), dataprovider.OrderASC, claims.Username) + + dataGetter := func(limit, offset int) ([]byte, int, error) { + shares, err := dataprovider.GetShares(limit, offset, dataprovider.OrderASC, claims.Username) if err != nil { - sendAPIResponse(w, r, err, getI18NErrorString(err, util.I18nError500Message), http.StatusInternalServerError) - return - } - shares = append(shares, sh...) - if len(sh) < defaultQueryLimit { - break + return nil, 0, err } + data, err := json.Marshal(shares) + return data, len(shares), err } - render.JSON(w, r, shares) + + streamJSONArray(w, defaultQueryLimit, dataGetter) } func (s *httpdServer) handleClientGetShares(w http.ResponseWriter, r *http.Request) {