beenull/sftpgo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

WebUI: add a JSON helper function

Browse source 
Signed-off-by: Nicola Murino <nicola.murino@gmail.com>
This commit is contained in:
Nicola Murino 2024-02-04 18:16:10 +01:00
parent c23d779280
commit e5836c8118
No known key found for this signature in database
GPG key ID: 935D2952DEC4EECF
7 changed files with 239 additions and 120 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
go.mod
View file

@ -52,7 +52,7 @@ require (
github.com/robfig/cron/v3 v3.0.1
github.com/rs/cors v1.10.1
github.com/rs/xid v1.5.0
github.com/rs/zerolog v1.31.0
github.com/rs/zerolog v1.32.0
github.com/sftpgo/sdk v0.1.6-0.20240114195211-3f4916cc829c
github.com/shirou/gopsutil/v3 v3.24.1
github.com/spf13/afero v1.11.0

4
go.sum
View file

@ -338,8 +338,8 @@ github.com/rs/cors v1.10.1 h1:L0uuZVXIKlI1SShY2nhFfo44TYvDPQ1w4oFkUJNfhyo=
github.com/rs/cors v1.10.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A=
github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
github.com/rs/zerolog v1.32.0 h1:keLypqrlIjaFsbmJOBdB/qvyF8KEtCWHwobLp5l/mQ0=
github.com/rs/zerolog v1.32.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ=

35
internal/httpd/api_utils.go
View file

@ -299,6 +299,41 @@ func renderAPIDirContents(w http.ResponseWriter, r *http.Request, contents []os.
render.JSON(w, r, results)
}
func streamData(w io.Writer, data []byte) {
b := bytes.NewBuffer(data)
_, err := io.CopyN(w, b, int64(len(data)))
if err != nil {
panic(http.ErrAbortHandler)
}
}
func streamJSONArray(w http.ResponseWriter, chunkSize int, dataGetter func(limit, offset int) ([]byte, int, error)) {
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Accept-Ranges", "none")
w.WriteHeader(http.StatusOK)
streamData(w, []byte("["))
offset := 0
for {
data, count, err := dataGetter(chunkSize, offset)
if err != nil {
panic(http.ErrAbortHandler)
}
if count == 0 {
break
}
if offset > 0 {
streamData(w, []byte(","))
}
streamData(w, data[1:len(data)-1])
if count < chunkSize {
break
}
offset += count
}
streamData(w, []byte("]"))
}
func getCompressedFileName(username string, files []string) string {
if len(files) == 1 {
name := path.Base(files[0])

137
internal/httpd/httpd_test.go
View file

@ -7014,11 +7014,18 @@ func TestProviderErrors(t *testing.T) {
assert.Equal(t, http.StatusOK, rr.Code)
assert.Contains(t, rr.Body.String(), util.I18nErrorGetUser)
req, err = http.NewRequest(http.MethodGet, webClientSharesPath+jsonAPISuffix, nil)
assert.NoError(t, err)
setJWTCookieForReq(req, userWebToken)
rr = executeRequest(req)
checkResponseCode(t, http.StatusInternalServerError, rr)
getJSONShares := func() {
defer func() {
rcv := recover()
assert.Equal(t, http.ErrAbortHandler, rcv)
}()
req, err := http.NewRequest(http.MethodGet, webClientSharesPath+jsonAPISuffix, nil)
assert.NoError(t, err)
setJWTCookieForReq(req, userWebToken)
executeRequest(req)
}
getJSONShares()
req, err = http.NewRequest(http.MethodGet, webClientSharePath, nil)
assert.NoError(t, err)
setJWTCookieForReq(req, userWebToken)
@ -7256,11 +7263,19 @@ func TestProviderErrors(t *testing.T) {
setJWTCookieForReq(req, testServerToken)
rr = executeRequest(req)
checkResponseCode(t, http.StatusInternalServerError, rr)
req, err = http.NewRequest(http.MethodGet, webAdminEventActionsPath+jsonAPISuffix, nil)
assert.NoError(t, err)
setJWTCookieForReq(req, testServerToken)
rr = executeRequest(req)
checkResponseCode(t, http.StatusInternalServerError, rr)
getJSONActions := func() {
defer func() {
rcv := recover()
assert.Equal(t, http.ErrAbortHandler, rcv)
}()
req, err := http.NewRequest(http.MethodGet, webAdminEventActionsPath+jsonAPISuffix, nil)
assert.NoError(t, err)
setJWTCookieForReq(req, testServerToken)
executeRequest(req)
}
getJSONActions()
req, err = http.NewRequest(http.MethodGet, path.Join(webAdminEventRulePath, "rulename"), nil)
assert.NoError(t, err)
setJWTCookieForReq(req, testServerToken)
@ -7271,11 +7286,19 @@ func TestProviderErrors(t *testing.T) {
setJWTCookieForReq(req, testServerToken)
rr = executeRequest(req)
checkResponseCode(t, http.StatusInternalServerError, rr)
req, err = http.NewRequest(http.MethodGet, webAdminEventRulesPath+jsonAPISuffix+"?qlimit=10", nil)
assert.NoError(t, err)
setJWTCookieForReq(req, testServerToken)
rr = executeRequest(req)
checkResponseCode(t, http.StatusInternalServerError, rr)
getJSONRules := func() {
defer func() {
rcv := recover()
assert.Equal(t, http.ErrAbortHandler, rcv)
}()
req, err := http.NewRequest(http.MethodGet, webAdminEventRulesPath+jsonAPISuffix, nil)
assert.NoError(t, err)
setJWTCookieForReq(req, testServerToken)
executeRequest(req)
}
getJSONRules()
req, err = http.NewRequest(http.MethodGet, webAdminEventRulePath, nil)
assert.NoError(t, err)
setJWTCookieForReq(req, testServerToken)
@ -18525,7 +18548,7 @@ func TestWebUserShare(t *testing.T) {
rr = executeRequest(req)
checkResponseCode(t, http.StatusOK, rr)
req, err = http.NewRequest(http.MethodGet, webClientSharesPath+jsonAPISuffix, nil) //nolint:goconst
req, err = http.NewRequest(http.MethodGet, webClientSharesPath+jsonAPISuffix, nil)
assert.NoError(t, err)
req.RemoteAddr = defaultRemoteAddr
setJWTCookieForReq(req, token)
@ -22878,6 +22901,13 @@ func TestWebEventAction(t *testing.T) {
setCSRFHeaderForReq(req, csrfToken)
rr = executeRequest(req)
checkResponseCode(t, http.StatusOK, rr)
req, err = http.NewRequest(http.MethodGet, webAdminEventActionsPath+jsonAPISuffix, nil)
assert.NoError(t, err)
setJWTCookieForReq(req, webToken)
rr = executeRequest(req)
checkResponseCode(t, http.StatusOK, rr)
assert.Equal(t, `[]`, rr.Body.String())
}
func TestWebEventRule(t *testing.T) {
@ -24928,18 +24958,39 @@ func TestProviderClosedMock(t *testing.T) {
rr = executeRequest(req)
checkResponseCode(t, http.StatusInternalServerError, rr)
req, _ = http.NewRequest(http.MethodGet, webFoldersPath+jsonAPISuffix, nil)
setJWTCookieForReq(req, token)
rr = executeRequest(req)
checkResponseCode(t, http.StatusInternalServerError, rr)
req, _ = http.NewRequest(http.MethodGet, webGroupsPath+jsonAPISuffix, nil)
setJWTCookieForReq(req, token)
rr = executeRequest(req)
checkResponseCode(t, http.StatusInternalServerError, rr)
req, _ = http.NewRequest(http.MethodGet, webUsersPath+jsonAPISuffix, nil)
setJWTCookieForReq(req, token)
rr = executeRequest(req)
checkResponseCode(t, http.StatusInternalServerError, rr)
getJSONFolders := func() {
defer func() {
rcv := recover()
assert.Equal(t, http.ErrAbortHandler, rcv)
}()
req, _ := http.NewRequest(http.MethodGet, webFoldersPath+jsonAPISuffix, nil)
setJWTCookieForReq(req, token)
executeRequest(req)
}
getJSONFolders()
getJSONGroups := func() {
defer func() {
rcv := recover()
assert.Equal(t, http.ErrAbortHandler, rcv)
}()
req, _ := http.NewRequest(http.MethodGet, webGroupsPath+jsonAPISuffix, nil)
setJWTCookieForReq(req, token)
executeRequest(req)
}
getJSONGroups()
getJSONUsers := func() {
defer func() {
rcv := recover()
assert.Equal(t, http.ErrAbortHandler, rcv)
}()
req, _ := http.NewRequest(http.MethodGet, webUsersPath+jsonAPISuffix, nil)
setJWTCookieForReq(req, token)
executeRequest(req)
}
getJSONUsers()
req, _ = http.NewRequest(http.MethodGet, webUserPath+"/0", nil)
setJWTCookieForReq(req, token)
rr = executeRequest(req)
@ -24961,10 +25012,16 @@ func TestProviderClosedMock(t *testing.T) {
rr = executeRequest(req)
checkResponseCode(t, http.StatusInternalServerError, rr)
req, _ = http.NewRequest(http.MethodGet, webAdminsPath+jsonAPISuffix, nil)
setJWTCookieForReq(req, token)
rr = executeRequest(req)
checkResponseCode(t, http.StatusInternalServerError, rr)
getJSONAdmins := func() {
defer func() {
rcv := recover()
assert.Equal(t, http.ErrAbortHandler, rcv)
}()
req, _ := http.NewRequest(http.MethodGet, webAdminsPath+jsonAPISuffix, nil)
setJWTCookieForReq(req, token)
executeRequest(req)
}
getJSONAdmins()
req, _ = http.NewRequest(http.MethodGet, path.Join(webFolderPath, defaultTokenAuthUser), nil)
setJWTCookieForReq(req, token)
@ -24998,11 +25055,17 @@ func TestProviderClosedMock(t *testing.T) {
rr = executeRequest(req)
checkResponseCode(t, http.StatusInternalServerError, rr)
req, err = http.NewRequest(http.MethodGet, webAdminRolesPath+jsonAPISuffix, nil)
assert.NoError(t, err)
setJWTCookieForReq(req, token)
rr = executeRequest(req)
checkResponseCode(t, http.StatusInternalServerError, rr)
getJSONRoles := func() {
defer func() {
rcv := recover()
assert.Equal(t, http.ErrAbortHandler, rcv)
}()
req, err := http.NewRequest(http.MethodGet, webAdminRolesPath+jsonAPISuffix, nil)
assert.NoError(t, err)
setJWTCookieForReq(req, token)
executeRequest(req)
}
getJSONRoles()
req, err = http.NewRequest(http.MethodGet, path.Join(webAdminRolePath, role.Name), nil)
assert.NoError(t, err)

36
internal/httpd/internal_test.go
View file

@ -2195,6 +2195,33 @@ func TestRecoverer(t *testing.T) {
assert.Equal(t, http.StatusInternalServerError, rr.Code, rr.Body.String())
}
func TestStreamJSONArray(t *testing.T) {
dataGetter := func(limit, offset int) ([]byte, int, error) {
return nil, 0, nil
}
rr := httptest.NewRecorder()
streamJSONArray(rr, 10, dataGetter)
assert.Equal(t, `[]`, rr.Body.String())
data := []int{}
for i := 0; i < 10; i++ {
data = append(data, i)
}
dataGetter = func(limit, offset int) ([]byte, int, error) {
if offset >= len(data) {
return nil, 0, nil
}
val := data[offset]
data, err := json.Marshal([]int{val})
return data, 1, err
}
rr = httptest.NewRecorder()
streamJSONArray(rr, 1, dataGetter)
assert.Equal(t, `[0,1,2,3,4,5,6,7,8,9]`, rr.Body.String())
}
func TestCompressorAbortHandler(t *testing.T) {
defer func() {
rcv := recover()
@ -2209,6 +2236,15 @@ func TestCompressorAbortHandler(t *testing.T) {
renderCompressedFiles(&failingWriter{}, connection, "", nil, share)
}
func TestStreamDataAbortHandler(t *testing.T) {
defer func() {
rcv := recover()
assert.Equal(t, http.ErrAbortHandler, rcv)
}()
streamData(&failingWriter{}, []byte(`["a":"b"]`))
}
func TestZipErrors(t *testing.T) {
user := dataprovider.User{
BaseUser: sdk.BaseUser{

127
internal/httpd/webadmin.go
View file

@ -16,6 +16,7 @@ package httpd
import (
"context"
"encoding/json"
"errors"
"fmt"
"html/template"
@ -2878,19 +2879,17 @@ func getAllAdmins(w http.ResponseWriter, r *http.Request) {
sendAPIResponse(w, r, nil, util.I18nErrorInvalidToken, http.StatusForbidden)
return
}
admins := make([]dataprovider.Admin, 0, 50)
for {
a, err := dataprovider.GetAdmins(defaultQueryLimit, len(admins), dataprovider.OrderASC)
dataGetter := func(limit, offset int) ([]byte, int, error) {
results, err := dataprovider.GetAdmins(limit, offset, dataprovider.OrderASC)
if err != nil {
sendAPIResponse(w, r, err, getI18NErrorString(err, util.I18nError500Message), http.StatusInternalServerError)
return
}
admins = append(admins, a...)
if len(a) < defaultQueryLimit {
break
return nil, 0, err
}
data, err := json.Marshal(results)
return data, len(results), err
}
render.JSON(w, r, admins)
streamJSONArray(w, defaultQueryLimit, dataGetter)
}
func (s *httpdServer) handleGetWebAdmins(w http.ResponseWriter, r *http.Request) {
@ -3043,19 +3042,17 @@ func getAllUsers(w http.ResponseWriter, r *http.Request) {
sendAPIResponse(w, r, nil, util.I18nErrorInvalidToken, http.StatusForbidden)
return
}
users := make([]dataprovider.User, 0, 100)
for {
u, err := dataprovider.GetUsers(defaultQueryLimit, len(users), dataprovider.OrderASC, claims.Role)
dataGetter := func(limit, offset int) ([]byte, int, error) {
results, err := dataprovider.GetUsers(limit, offset, dataprovider.OrderASC, claims.Role)
if err != nil {
sendAPIResponse(w, r, err, getI18NErrorString(err, util.I18nError500Message), http.StatusInternalServerError)
return
}
users = append(users, u...)
if len(u) < defaultQueryLimit {
break
return nil, 0, err
}
data, err := json.Marshal(results)
return data, len(results), err
}
render.JSON(w, r, users)
streamJSONArray(w, defaultQueryLimit, dataGetter)
}
func (s *httpdServer) handleGetWebUsers(w http.ResponseWriter, r *http.Request) {
@ -3538,19 +3535,17 @@ func (s *httpdServer) getWebVirtualFolders(w http.ResponseWriter, r *http.Reques
func getAllFolders(w http.ResponseWriter, r *http.Request) {
r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize)
folders := make([]vfs.BaseVirtualFolder, 0, 50)
for {
f, err := dataprovider.GetFolders(defaultQueryLimit, len(folders), dataprovider.OrderASC, false)
dataGetter := func(limit, offset int) ([]byte, int, error) {
results, err := dataprovider.GetFolders(limit, offset, dataprovider.OrderASC, false)
if err != nil {
sendAPIResponse(w, r, err, getI18NErrorString(err, util.I18nError500Message), http.StatusInternalServerError)
return
}
folders = append(folders, f...)
if len(f) < defaultQueryLimit {
break
return nil, 0, err
}
data, err := json.Marshal(results)
return data, len(results), err
}
render.JSON(w, r, folders)
streamJSONArray(w, defaultQueryLimit, dataGetter)
}
func (s *httpdServer) handleWebGetFolders(w http.ResponseWriter, r *http.Request) {
@ -3578,19 +3573,17 @@ func (s *httpdServer) getWebGroups(w http.ResponseWriter, r *http.Request, limit
func getAllGroups(w http.ResponseWriter, r *http.Request) {
r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize)
groups := make([]dataprovider.Group, 0, 50)
for {
f, err := dataprovider.GetGroups(defaultQueryLimit, len(groups), dataprovider.OrderASC, false)
dataGetter := func(limit, offset int) ([]byte, int, error) {
results, err := dataprovider.GetGroups(limit, offset, dataprovider.OrderASC, false)
if err != nil {
sendAPIResponse(w, r, err, getI18NErrorString(err, util.I18nError500Message), http.StatusInternalServerError)
return
}
groups = append(groups, f...)
if len(f) < defaultQueryLimit {
break
return nil, 0, err
}
data, err := json.Marshal(results)
return data, len(results), err
}
render.JSON(w, r, groups)
streamJSONArray(w, defaultQueryLimit, dataGetter)
}
func (s *httpdServer) handleWebGetGroups(w http.ResponseWriter, r *http.Request) {
@ -3707,19 +3700,17 @@ func (s *httpdServer) getWebEventActions(w http.ResponseWriter, r *http.Request,
func getAllActions(w http.ResponseWriter, r *http.Request) {
r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize)
actions := make([]dataprovider.BaseEventAction, 0, 10)
for {
res, err := dataprovider.GetEventActions(defaultQueryLimit, len(actions), dataprovider.OrderASC, false)
dataGetter := func(limit, offset int) ([]byte, int, error) {
results, err := dataprovider.GetEventActions(limit, offset, dataprovider.OrderASC, false)
if err != nil {
sendAPIResponse(w, r, err, getI18NErrorString(err, util.I18nError500Message), http.StatusInternalServerError)
return
}
actions = append(actions, res...)
if len(res) < defaultQueryLimit {
break
return nil, 0, err
}
data, err := json.Marshal(results)
return data, len(results), err
}
render.JSON(w, r, actions)
streamJSONArray(w, defaultQueryLimit, dataGetter)
}
func (s *httpdServer) handleWebGetEventActions(w http.ResponseWriter, r *http.Request) {
@ -3819,19 +3810,17 @@ func (s *httpdServer) handleWebUpdateEventActionPost(w http.ResponseWriter, r *h
func getAllRules(w http.ResponseWriter, r *http.Request) {
r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize)
rules := make([]dataprovider.EventRule, 0, 10)
for {
res, err := dataprovider.GetEventRules(defaultQueryLimit, len(rules), dataprovider.OrderASC)
dataGetter := func(limit, offset int) ([]byte, int, error) {
results, err := dataprovider.GetEventRules(limit, offset, dataprovider.OrderASC)
if err != nil {
sendAPIResponse(w, r, err, getI18NErrorString(err, util.I18nError500Message), http.StatusInternalServerError)
return
}
rules = append(rules, res...)
if len(res) < defaultQueryLimit {
break
return nil, 0, err
}
data, err := json.Marshal(results)
return data, len(results), err
}
render.JSON(w, r, rules)
streamJSONArray(w, defaultQueryLimit, dataGetter)
}
func (s *httpdServer) handleWebGetEventRules(w http.ResponseWriter, r *http.Request) {
@ -3942,19 +3931,17 @@ func (s *httpdServer) getWebRoles(w http.ResponseWriter, r *http.Request, limit
func getAllRoles(w http.ResponseWriter, r *http.Request) {
r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize)
roles := make([]dataprovider.Role, 0, 10)
for {
res, err := dataprovider.GetRoles(defaultQueryLimit, len(roles), dataprovider.OrderASC, false)
dataGetter := func(limit, offset int) ([]byte, int, error) {
results, err := dataprovider.GetRoles(limit, offset, dataprovider.OrderASC, false)
if err != nil {
sendAPIResponse(w, r, err, getI18NErrorString(err, util.I18nError500Message), http.StatusInternalServerError)
return
}
roles = append(roles, res...)
if len(res) < defaultQueryLimit {
break
return nil, 0, err
}
data, err := json.Marshal(results)
return data, len(results), err
}
render.JSON(w, r, roles)
streamJSONArray(w, defaultQueryLimit, dataGetter)
}
func (s *httpdServer) handleWebGetRoles(w http.ResponseWriter, r *http.Request) {

18
internal/httpd/webclient.go
View file

@ -1521,19 +1521,17 @@ func getAllShares(w http.ResponseWriter, r *http.Request) {
sendAPIResponse(w, r, nil, util.I18nErrorInvalidToken, http.StatusForbidden)
return
}
shares := make([]dataprovider.Share, 0, 10)
for {
sh, err := dataprovider.GetShares(defaultQueryLimit, len(shares), dataprovider.OrderASC, claims.Username)
dataGetter := func(limit, offset int) ([]byte, int, error) {
shares, err := dataprovider.GetShares(limit, offset, dataprovider.OrderASC, claims.Username)
if err != nil {
sendAPIResponse(w, r, err, getI18NErrorString(err, util.I18nError500Message), http.StatusInternalServerError)
return
}
shares = append(shares, sh...)
if len(sh) < defaultQueryLimit {
break
return nil, 0, err
}
data, err := json.Marshal(shares)
return data, len(shares), err
}
render.JSON(w, r, shares)
streamJSONArray(w, defaultQueryLimit, dataGetter)
}
func (s *httpdServer) handleClientGetShares(w http.ResponseWriter, r *http.Request) {