From 712f2053a444c23ccaa89b9d5b7133fc118e285d Mon Sep 17 00:00:00 2001 From: Nicola Murino Date: Tue, 18 Apr 2023 18:11:23 +0200 Subject: [PATCH] REST API dumpdata: allow to specify the resources to dump Signed-off-by: Nicola Murino --- .github/workflows/development.yml | 26 ++-- go.mod | 4 +- go.sum | 7 +- internal/common/eventmanager.go | 9 +- internal/dataprovider/dataprovider.go | 197 ++++++++++++++++++++------ internal/httpd/api_eventrule.go | 30 ++-- internal/httpd/api_folder.go | 15 +- internal/httpd/api_group.go | 15 +- internal/httpd/api_maintenance.go | 6 +- internal/httpd/api_user.go | 12 +- internal/httpd/api_utils.go | 7 + internal/httpd/httpd_test.go | 16 +++ internal/httpd/internal_test.go | 22 ++- internal/httpdtest/httpdtest.go | 5 +- openapi/openapi.yaml | 83 +++++++++++ 15 files changed, 363 insertions(+), 91 deletions(-) diff --git a/.github/workflows/development.yml b/.github/workflows/development.yml index 697ed854..a61d6e6d 100644 --- a/.github/workflows/development.yml +++ b/.github/workflows/development.yml @@ -308,19 +308,6 @@ jobs: go build -trimpath -ldflags "-s -w" -o ipfilter cd - - - name: Run tests using PostgreSQL provider - run: | - ./sftpgo initprovider - ./sftpgo resetprovider --force - go test -v -tags nopgxregisterdefaulttypes -p 1 -timeout 15m ./... -covermode=atomic - env: - SFTPGO_DATA_PROVIDER__DRIVER: postgresql - SFTPGO_DATA_PROVIDER__NAME: sftpgo - SFTPGO_DATA_PROVIDER__HOST: localhost - SFTPGO_DATA_PROVIDER__PORT: 5432 - SFTPGO_DATA_PROVIDER__USERNAME: postgres - SFTPGO_DATA_PROVIDER__PASSWORD: postgres - - name: Run tests using MySQL provider run: | ./sftpgo initprovider @@ -334,6 +321,19 @@ jobs: SFTPGO_DATA_PROVIDER__USERNAME: sftpgo SFTPGO_DATA_PROVIDER__PASSWORD: sftpgo + - name: Run tests using PostgreSQL provider + run: | + ./sftpgo initprovider + ./sftpgo resetprovider --force + go test -v -tags nopgxregisterdefaulttypes -p 1 -timeout 15m ./... -covermode=atomic + env: + SFTPGO_DATA_PROVIDER__DRIVER: postgresql + SFTPGO_DATA_PROVIDER__NAME: sftpgo + SFTPGO_DATA_PROVIDER__HOST: localhost + SFTPGO_DATA_PROVIDER__PORT: 5432 + SFTPGO_DATA_PROVIDER__USERNAME: postgres + SFTPGO_DATA_PROVIDER__PASSWORD: postgres + - name: Run tests using MariaDB provider run: | ./sftpgo initprovider diff --git a/go.mod b/go.mod index a4507d18..34903313 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.2 github.com/jackc/pgx/v5 v5.3.2-0.20230411230705-2cf1541bb90a github.com/jlaffaye/ftp v0.0.0-20201112195030-9aae4d151126 - github.com/klauspost/compress v1.16.4 + github.com/klauspost/compress v1.16.5 github.com/lestrrat-go/jwx/v2 v2.0.9 github.com/lithammer/shortuuid/v3 v3.0.7 github.com/mattn/go-sqlite3 v1.14.16 @@ -99,7 +99,7 @@ require ( github.com/aws/smithy-go v1.13.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/boombuler/barcode v1.0.1 // indirect - github.com/cenkalti/backoff/v4 v4.2.0 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect diff --git a/go.sum b/go.sum index 3201f517..68633aab 100644 --- a/go.sum +++ b/go.sum @@ -653,8 +653,9 @@ github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3k github.com/casbin/casbin/v2 v2.37.0/go.mod h1:vByNa/Fchek0KZUgG5wEsl7iFsiviAYKRtgrQfcJqHg= github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= -github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= @@ -1442,8 +1443,8 @@ github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdY github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.16.4 h1:91KN02FnsOYhuunwU4ssRe8lc2JosWmizWa91B5v1PU= -github.com/klauspost/compress v1.16.4/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= +github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= diff --git a/internal/common/eventmanager.go b/internal/common/eventmanager.go index ef6531c7..3c633153 100644 --- a/internal/common/eventmanager.go +++ b/internal/common/eventmanager.go @@ -631,12 +631,12 @@ func (p *EventParams) getStatusString() string { // getUsers returns users with group settings not applied func (p *EventParams) getUsers() ([]dataprovider.User, error) { if p.sender == "" { - users, err := dataprovider.DumpUsers() + dump, err := dataprovider.DumpData([]string{dataprovider.DumpScopeUsers}) if err != nil { eventManagerLog(logger.LevelError, "unable to get users: %+v", err) - return users, errors.New("unable to get users") + return nil, errors.New("unable to get users") } - return users, nil + return dump.Users, nil } user, err := p.getUserFromSender() if err != nil { @@ -668,7 +668,8 @@ func (p *EventParams) getUserFromSender() (dataprovider.User, error) { func (p *EventParams) getFolders() ([]vfs.BaseVirtualFolder, error) { if p.sender == "" { - return dataprovider.DumpFolders() + dump, err := dataprovider.DumpData([]string{dataprovider.DumpScopeFolders}) + return dump.Folders, err } folder, err := dataprovider.GetFolderByName(p.sender) if err != nil { diff --git a/internal/dataprovider/dataprovider.go b/internal/dataprovider/dataprovider.go index 3a619ec8..4da37155 100644 --- a/internal/dataprovider/dataprovider.go +++ b/internal/dataprovider/dataprovider.go @@ -130,6 +130,21 @@ const ( protocolHTTP = "HTTP" ) +// Dump scopes +const ( + DumpScopeUsers = "users" + DumpScopeFolders = "folders" + DumpScopeGroups = "groups" + DumpScopeAdmins = "admins" + DumpScopeAPIKeys = "api_keys" + DumpScopeShares = "shares" + DumpScopeActions = "actions" + DumpScopeRules = "rules" + DumpScopeRoles = "roles" + DumpScopeIPLists = "ip_lists" + DumpScopeConfigs = "configs" +) + var ( // SupportedProviders defines the supported data providers SupportedProviders = []string{SQLiteDataProviderName, PGSQLDataProviderName, MySQLDataProviderName, @@ -541,7 +556,7 @@ func (c *Config) doBackup() (string, error) { providerLog(logger.LevelError, "unable to create backup dir %q: %v", outputFile, err) return outputFile, fmt.Errorf("unable to create backup dir: %w", err) } - backup, err := DumpData() + backup, err := DumpData(nil) if err != nil { providerLog(logger.LevelError, "unable to execute backup: %v", err) return outputFile, fmt.Errorf("unable to dump backup data: %w", err) @@ -2289,76 +2304,168 @@ func GetFolders(limit, offset int, order string, minimal bool) ([]vfs.BaseVirtua return provider.getFolders(limit, offset, order, minimal) } -// DumpUsers returns all users, including confidential data -func DumpUsers() ([]User, error) { - return provider.dumpUsers() +func dumpUsers(data *BackupData, scopes []string) error { + if len(scopes) == 0 || util.Contains(scopes, DumpScopeUsers) { + users, err := provider.dumpUsers() + if err != nil { + return err + } + data.Users = users + } + return nil } -// DumpFolders returns all folders, including confidential data -func DumpFolders() ([]vfs.BaseVirtualFolder, error) { - return provider.dumpFolders() +func dumpFolders(data *BackupData, scopes []string) error { + if len(scopes) == 0 || util.Contains(scopes, DumpScopeFolders) { + folders, err := provider.dumpFolders() + if err != nil { + return err + } + data.Folders = folders + } + return nil } -// DumpData returns all users, groups, folders, admins, api keys, shares, actions, rules -func DumpData() (BackupData, error) { - var data BackupData - groups, err := provider.dumpGroups() - if err != nil { +func dumpGroups(data *BackupData, scopes []string) error { + if len(scopes) == 0 || util.Contains(scopes, DumpScopeGroups) { + groups, err := provider.dumpGroups() + if err != nil { + return err + } + data.Groups = groups + } + return nil +} + +func dumpAdmins(data *BackupData, scopes []string) error { + if len(scopes) == 0 || util.Contains(scopes, DumpScopeAdmins) { + admins, err := provider.dumpAdmins() + if err != nil { + return err + } + data.Admins = admins + } + return nil +} + +func dumpAPIKeys(data *BackupData, scopes []string) error { + if len(scopes) == 0 || util.Contains(scopes, DumpScopeAPIKeys) { + apiKeys, err := provider.dumpAPIKeys() + if err != nil { + return err + } + data.APIKeys = apiKeys + } + return nil +} + +func dumpShares(data *BackupData, scopes []string) error { + if len(scopes) == 0 || util.Contains(scopes, DumpScopeShares) { + shares, err := provider.dumpShares() + if err != nil { + return err + } + data.Shares = shares + } + return nil +} + +func dumpActions(data *BackupData, scopes []string) error { + if len(scopes) == 0 || util.Contains(scopes, DumpScopeActions) { + actions, err := provider.dumpEventActions() + if err != nil { + return err + } + data.EventActions = actions + } + return nil +} + +func dumpRules(data *BackupData, scopes []string) error { + if len(scopes) == 0 || util.Contains(scopes, DumpScopeRules) { + rules, err := provider.dumpEventRules() + if err != nil { + return err + } + data.EventRules = rules + } + return nil +} + +func dumpRoles(data *BackupData, scopes []string) error { + if len(scopes) == 0 || util.Contains(scopes, DumpScopeRoles) { + roles, err := provider.dumpRoles() + if err != nil { + return err + } + data.Roles = roles + } + return nil +} + +func dumpIPLists(data *BackupData, scopes []string) error { + if len(scopes) == 0 || util.Contains(scopes, DumpScopeIPLists) { + ipLists, err := provider.dumpIPListEntries() + if err != nil { + return err + } + data.IPLists = ipLists + } + return nil +} + +func dumpConfigs(data *BackupData, scopes []string) error { + if len(scopes) == 0 || util.Contains(scopes, DumpScopeConfigs) { + configs, err := provider.getConfigs() + if err != nil { + return err + } + data.Configs = &configs + } + return nil +} + +// DumpData returns a dump containing the requested scopes. +// Empty scopes means all +func DumpData(scopes []string) (BackupData, error) { + data := BackupData{ + Version: DumpVersion, + } + if err := dumpGroups(&data, scopes); err != nil { return data, err } - users, err := provider.dumpUsers() - if err != nil { + if err := dumpUsers(&data, scopes); err != nil { return data, err } - folders, err := provider.dumpFolders() - if err != nil { + if err := dumpFolders(&data, scopes); err != nil { return data, err } - admins, err := provider.dumpAdmins() - if err != nil { + if err := dumpAdmins(&data, scopes); err != nil { return data, err } - apiKeys, err := provider.dumpAPIKeys() - if err != nil { + if err := dumpAPIKeys(&data, scopes); err != nil { return data, err } - shares, err := provider.dumpShares() - if err != nil { + if err := dumpShares(&data, scopes); err != nil { return data, err } - actions, err := provider.dumpEventActions() - if err != nil { + if err := dumpActions(&data, scopes); err != nil { return data, err } - rules, err := provider.dumpEventRules() - if err != nil { + if err := dumpRules(&data, scopes); err != nil { return data, err } - roles, err := provider.dumpRoles() - if err != nil { + if err := dumpRoles(&data, scopes); err != nil { return data, err } - ipLists, err := provider.dumpIPListEntries() - if err != nil { + if err := dumpIPLists(&data, scopes); err != nil { return data, err } - configs, err := provider.getConfigs() - if err != nil { + if err := dumpConfigs(&data, scopes); err != nil { return data, err } - data.Users = users - data.Groups = groups - data.Folders = folders - data.Admins = admins - data.APIKeys = apiKeys - data.Shares = shares - data.EventActions = actions - data.EventRules = rules - data.Roles = roles - data.IPLists = ipLists - data.Configs = &configs - data.Version = DumpVersion - return data, err + + return data, nil } // ParseDumpData tries to parse data as BackupData diff --git a/internal/httpd/api_eventrule.go b/internal/httpd/api_eventrule.go index cb8dc40f..9440520e 100644 --- a/internal/httpd/api_eventrule.go +++ b/internal/httpd/api_eventrule.go @@ -42,13 +42,15 @@ func getEventActions(w http.ResponseWriter, r *http.Request) { render.JSON(w, r, actions) } -func renderEventAction(w http.ResponseWriter, r *http.Request, name string, status int) { +func renderEventAction(w http.ResponseWriter, r *http.Request, name string, claims *jwtTokenClaims, status int) { action, err := dataprovider.EventActionExists(name) if err != nil { sendAPIResponse(w, r, err, "", getRespStatus(err)) return } - action.PrepareForRendering() + if hideConfidentialData(claims, r) { + action.PrepareForRendering() + } if status != http.StatusOK { ctx := context.WithValue(r.Context(), render.StatusCtxKey, status) render.JSON(w, r.WithContext(ctx), action) @@ -59,8 +61,13 @@ func renderEventAction(w http.ResponseWriter, r *http.Request, name string, stat func getEventActionByName(w http.ResponseWriter, r *http.Request) { r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize) + claims, err := getTokenClaims(r) + if err != nil || claims.Username == "" { + sendAPIResponse(w, r, err, "Invalid token claims", http.StatusBadRequest) + return + } name := getURLParam(r, "name") - renderEventAction(w, r, name, http.StatusOK) + renderEventAction(w, r, name, &claims, http.StatusOK) } func addEventAction(w http.ResponseWriter, r *http.Request) { @@ -84,7 +91,7 @@ func addEventAction(w http.ResponseWriter, r *http.Request) { return } w.Header().Add("Location", fmt.Sprintf("%s/%s", eventActionsPath, url.PathEscape(action.Name))) - renderEventAction(w, r, action.Name, http.StatusCreated) + renderEventAction(w, r, action.Name, &claims, http.StatusCreated) } func updateEventAction(w http.ResponseWriter, r *http.Request) { @@ -158,13 +165,15 @@ func getEventRules(w http.ResponseWriter, r *http.Request) { render.JSON(w, r, rules) } -func renderEventRule(w http.ResponseWriter, r *http.Request, name string, status int) { +func renderEventRule(w http.ResponseWriter, r *http.Request, name string, claims *jwtTokenClaims, status int) { rule, err := dataprovider.EventRuleExists(name) if err != nil { sendAPIResponse(w, r, err, "", getRespStatus(err)) return } - rule.PrepareForRendering() + if hideConfidentialData(claims, r) { + rule.PrepareForRendering() + } if status != http.StatusOK { ctx := context.WithValue(r.Context(), render.StatusCtxKey, status) render.JSON(w, r.WithContext(ctx), rule) @@ -175,8 +184,13 @@ func renderEventRule(w http.ResponseWriter, r *http.Request, name string, status func getEventRuleByName(w http.ResponseWriter, r *http.Request) { r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize) + claims, err := getTokenClaims(r) + if err != nil || claims.Username == "" { + sendAPIResponse(w, r, err, "Invalid token claims", http.StatusBadRequest) + return + } name := getURLParam(r, "name") - renderEventRule(w, r, name, http.StatusOK) + renderEventRule(w, r, name, &claims, http.StatusOK) } func addEventRule(w http.ResponseWriter, r *http.Request) { @@ -199,7 +213,7 @@ func addEventRule(w http.ResponseWriter, r *http.Request) { return } w.Header().Add("Location", fmt.Sprintf("%s/%s", eventRulesPath, url.PathEscape(rule.Name))) - renderEventRule(w, r, rule.Name, http.StatusCreated) + renderEventRule(w, r, rule.Name, &claims, http.StatusCreated) } func updateEventRule(w http.ResponseWriter, r *http.Request) { diff --git a/internal/httpd/api_folder.go b/internal/httpd/api_folder.go index ac9035b3..5bc72062 100644 --- a/internal/httpd/api_folder.go +++ b/internal/httpd/api_folder.go @@ -62,7 +62,7 @@ func addFolder(w http.ResponseWriter, r *http.Request) { return } w.Header().Add("Location", fmt.Sprintf("%s/%s", folderPath, url.PathEscape(folder.Name))) - renderFolder(w, r, folder.Name, http.StatusCreated) + renderFolder(w, r, folder.Name, &claims, http.StatusCreated) } func updateFolder(w http.ResponseWriter, r *http.Request) { @@ -103,13 +103,15 @@ func updateFolder(w http.ResponseWriter, r *http.Request) { sendAPIResponse(w, r, nil, "Folder updated", http.StatusOK) } -func renderFolder(w http.ResponseWriter, r *http.Request, name string, status int) { +func renderFolder(w http.ResponseWriter, r *http.Request, name string, claims *jwtTokenClaims, status int) { folder, err := dataprovider.GetFolderByName(name) if err != nil { sendAPIResponse(w, r, err, "", getRespStatus(err)) return } - folder.PrepareForRendering() + if hideConfidentialData(claims, r) { + folder.PrepareForRendering() + } if status != http.StatusOK { ctx := context.WithValue(r.Context(), render.StatusCtxKey, status) render.JSON(w, r.WithContext(ctx), folder) @@ -120,8 +122,13 @@ func renderFolder(w http.ResponseWriter, r *http.Request, name string, status in func getFolderByName(w http.ResponseWriter, r *http.Request) { r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize) + claims, err := getTokenClaims(r) + if err != nil || claims.Username == "" { + sendAPIResponse(w, r, err, "Invalid token claims", http.StatusBadRequest) + return + } name := getURLParam(r, "name") - renderFolder(w, r, name, http.StatusOK) + renderFolder(w, r, name, &claims, http.StatusOK) } func deleteFolder(w http.ResponseWriter, r *http.Request) { diff --git a/internal/httpd/api_group.go b/internal/httpd/api_group.go index ced86d4e..a74b2eca 100644 --- a/internal/httpd/api_group.go +++ b/internal/httpd/api_group.go @@ -61,7 +61,7 @@ func addGroup(w http.ResponseWriter, r *http.Request) { return } w.Header().Add("Location", fmt.Sprintf("%s/%s", groupPath, url.PathEscape(group.Name))) - renderGroup(w, r, group.Name, http.StatusCreated) + renderGroup(w, r, group.Name, &claims, http.StatusCreated) } func updateGroup(w http.ResponseWriter, r *http.Request) { @@ -111,13 +111,15 @@ func updateGroup(w http.ResponseWriter, r *http.Request) { sendAPIResponse(w, r, nil, "Group updated", http.StatusOK) } -func renderGroup(w http.ResponseWriter, r *http.Request, name string, status int) { +func renderGroup(w http.ResponseWriter, r *http.Request, name string, claims *jwtTokenClaims, status int) { group, err := dataprovider.GroupExists(name) if err != nil { sendAPIResponse(w, r, err, "", getRespStatus(err)) return } - group.PrepareForRendering() + if hideConfidentialData(claims, r) { + group.PrepareForRendering() + } if status != http.StatusOK { ctx := context.WithValue(r.Context(), render.StatusCtxKey, status) render.JSON(w, r.WithContext(ctx), group) @@ -128,8 +130,13 @@ func renderGroup(w http.ResponseWriter, r *http.Request, name string, status int func getGroupByName(w http.ResponseWriter, r *http.Request) { r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize) + claims, err := getTokenClaims(r) + if err != nil || claims.Username == "" { + sendAPIResponse(w, r, err, "Invalid token claims", http.StatusBadRequest) + return + } name := getURLParam(r, "name") - renderGroup(w, r, name, http.StatusOK) + renderGroup(w, r, name, &claims, http.StatusOK) } func deleteGroup(w http.ResponseWriter, r *http.Request) { diff --git a/internal/httpd/api_maintenance.go b/internal/httpd/api_maintenance.go index a542f2d7..b8db1b3d 100644 --- a/internal/httpd/api_maintenance.go +++ b/internal/httpd/api_maintenance.go @@ -51,6 +51,7 @@ func validateBackupFile(outputFile string) (string, error) { func dumpData(w http.ResponseWriter, r *http.Request) { r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize) var outputFile, outputData, indent string + var scopes []string if _, ok := r.URL.Query()["output-file"]; ok { outputFile = strings.TrimSpace(r.URL.Query().Get("output-file")) } @@ -60,6 +61,9 @@ func dumpData(w http.ResponseWriter, r *http.Request) { if _, ok := r.URL.Query()["indent"]; ok { indent = strings.TrimSpace(r.URL.Query().Get("indent")) } + if _, ok := r.URL.Query()["scopes"]; ok { + scopes = getCommaSeparatedQueryParam(r, "scopes") + } if outputData != "1" { var err error @@ -78,7 +82,7 @@ func dumpData(w http.ResponseWriter, r *http.Request) { logger.Debug(logSender, "", "dumping data to: %q", outputFile) } - backup, err := dataprovider.DumpData() + backup, err := dataprovider.DumpData(scopes) if err != nil { logger.Error(logSender, "", "dumping data error: %v, output file: %q", err, outputFile) sendAPIResponse(w, r, err, "", getRespStatus(err)) diff --git a/internal/httpd/api_user.go b/internal/httpd/api_user.go index b780e9fc..136a700e 100644 --- a/internal/httpd/api_user.go +++ b/internal/httpd/api_user.go @@ -62,16 +62,18 @@ func getUserByUsername(w http.ResponseWriter, r *http.Request) { return } username := getURLParam(r, "username") - renderUser(w, r, username, claims.Role, http.StatusOK) + renderUser(w, r, username, &claims, http.StatusOK) } -func renderUser(w http.ResponseWriter, r *http.Request, username, role string, status int) { - user, err := dataprovider.UserExists(username, role) +func renderUser(w http.ResponseWriter, r *http.Request, username string, claims *jwtTokenClaims, status int) { + user, err := dataprovider.UserExists(username, claims.Role) if err != nil { sendAPIResponse(w, r, err, "", getRespStatus(err)) return } - user.PrepareForRendering() + if hideConfidentialData(claims, r) { + user.PrepareForRendering() + } if status != http.StatusOK { ctx := context.WithValue(r.Context(), render.StatusCtxKey, status) render.JSON(w, r.WithContext(ctx), user) @@ -116,7 +118,7 @@ func addUser(w http.ResponseWriter, r *http.Request) { return } w.Header().Add("Location", fmt.Sprintf("%s/%s", userPath, url.PathEscape(user.Username))) - renderUser(w, r, user.Username, claims.Role, http.StatusCreated) + renderUser(w, r, user.Username, &claims, http.StatusCreated) } func disableUser2FA(w http.ResponseWriter, r *http.Request) { diff --git a/internal/httpd/api_utils.go b/internal/httpd/api_utils.go index 72824e03..349c86d4 100644 --- a/internal/httpd/api_utils.go +++ b/internal/httpd/api_utils.go @@ -771,3 +771,10 @@ func getProtocolFromRequest(r *http.Request) string { } return common.ProtocolHTTP } + +func hideConfidentialData(claims *jwtTokenClaims, r *http.Request) bool { + if !claims.hasPerm(dataprovider.PermAdminManageSystem) { + return true + } + return r.URL.Query().Get("confidential_data") != "1" +} diff --git a/internal/httpd/httpd_test.go b/internal/httpd/httpd_test.go index 69cad376..df1e140c 100644 --- a/internal/httpd/httpd_test.go +++ b/internal/httpd/httpd_test.go @@ -7972,6 +7972,22 @@ func TestLoaddata(t *testing.T) { assert.Equal(t, int64(789), folder.LastQuotaUpdate) assert.Equal(t, folderDesc, folder.Description) assert.Len(t, folder.Users, 1) + response, _, err = httpdtest.Dumpdata("", "1", "0", http.StatusOK, dataprovider.DumpScopeUsers) + assert.NoError(t, err) + dumpedData = dataprovider.BackupData{} + data, err = json.Marshal(response) + assert.NoError(t, err) + err = json.Unmarshal(data, &dumpedData) + assert.NoError(t, err) + assert.Greater(t, len(dumpedData.Users), 0) + assert.Len(t, dumpedData.Admins, 0) + assert.Len(t, dumpedData.Folders, 0) + assert.Len(t, dumpedData.Groups, 0) + assert.Len(t, dumpedData.Roles, 0) + assert.Len(t, dumpedData.EventRules, 0) + assert.Len(t, dumpedData.EventActions, 0) + assert.Len(t, dumpedData.IPLists, 0) + _, err = httpdtest.RemoveFolder(folder, http.StatusOK) assert.NoError(t, err) _, err = httpdtest.RemoveUser(user, http.StatusOK) diff --git a/internal/httpd/internal_test.go b/internal/httpd/internal_test.go index 7861f0e0..72f2c57e 100644 --- a/internal/httpd/internal_test.go +++ b/internal/httpd/internal_test.go @@ -600,6 +600,11 @@ func TestInvalidToken(t *testing.T) { assert.Equal(t, http.StatusBadRequest, rr.Code) assert.Contains(t, rr.Body.String(), "Invalid token claims") + rr = httptest.NewRecorder() + getFolderByName(rr, req) + assert.Equal(t, http.StatusBadRequest, rr.Code) + assert.Contains(t, rr.Body.String(), "Invalid token claims") + rr = httptest.NewRecorder() deleteFolder(rr, req) assert.Equal(t, http.StatusBadRequest, rr.Code) @@ -660,6 +665,11 @@ func TestInvalidToken(t *testing.T) { assert.Equal(t, http.StatusBadRequest, rr.Code) assert.Contains(t, rr.Body.String(), "Invalid token claims") + rr = httptest.NewRecorder() + getGroupByName(rr, req) + assert.Equal(t, http.StatusBadRequest, rr.Code) + assert.Contains(t, rr.Body.String(), "Invalid token claims") + rr = httptest.NewRecorder() deleteGroup(rr, req) assert.Equal(t, http.StatusBadRequest, rr.Code) @@ -670,6 +680,11 @@ func TestInvalidToken(t *testing.T) { assert.Equal(t, http.StatusBadRequest, rr.Code) assert.Contains(t, rr.Body.String(), "Invalid token claims") + rr = httptest.NewRecorder() + getEventActionByName(rr, req) + assert.Equal(t, http.StatusBadRequest, rr.Code) + assert.Contains(t, rr.Body.String(), "Invalid token claims") + rr = httptest.NewRecorder() updateEventAction(rr, req) assert.Equal(t, http.StatusBadRequest, rr.Code) @@ -680,6 +695,11 @@ func TestInvalidToken(t *testing.T) { assert.Equal(t, http.StatusBadRequest, rr.Code) assert.Contains(t, rr.Body.String(), "Invalid token claims") + rr = httptest.NewRecorder() + getEventRuleByName(rr, req) + assert.Equal(t, http.StatusBadRequest, rr.Code) + assert.Contains(t, rr.Body.String(), "Invalid token claims") + rr = httptest.NewRecorder() addEventRule(rr, req) assert.Equal(t, http.StatusBadRequest, rr.Code) @@ -1606,7 +1626,7 @@ func TestChangePwdValidationErrors(t *testing.T) { func TestRenderUnexistingFolder(t *testing.T) { rr := httptest.NewRecorder() req, _ := http.NewRequest(http.MethodPost, folderPath, nil) - renderFolder(rr, req, "path not mapped", http.StatusOK) + renderFolder(rr, req, "path not mapped", &jwtTokenClaims{}, http.StatusOK) assert.Equal(t, http.StatusNotFound, rr.Code) } diff --git a/internal/httpdtest/httpdtest.go b/internal/httpdtest/httpdtest.go index 8e5d3508..b4b71589 100644 --- a/internal/httpdtest/httpdtest.go +++ b/internal/httpdtest/httpdtest.go @@ -1470,7 +1470,7 @@ func RemoveDefenderHostByIP(ip string, expectedStatusCode int) ([]byte, error) { // Dumpdata requests a backup to outputFile. // outputFile is relative to the configured backups_path -func Dumpdata(outputFile, outputData, indent string, expectedStatusCode int) (map[string]any, []byte, error) { +func Dumpdata(outputFile, outputData, indent string, expectedStatusCode int, scopes ...string) (map[string]any, []byte, error) { var response map[string]any var body []byte url, err := url.Parse(buildURLRelativeToBase(dumpDataPath)) @@ -1487,6 +1487,9 @@ func Dumpdata(outputFile, outputData, indent string, expectedStatusCode int) (ma if indent != "" { q.Add("indent", indent) } + if len(scopes) > 0 { + q.Add("scopes", strings.Join(scopes, ",")) + } url.RawQuery = q.Encode() resp, err := sendHTTPRequest(http.MethodGet, url.String(), nil, "", getDefaultToken()) if err != nil { diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index 9f8a8e5f..09802be2 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -1571,6 +1571,12 @@ paths: summary: Add folder operationId: add_folder description: Adds a new folder. A quota scan is required to update the used files/size + parameters: + - in: query + name: confidential_data + schema: + type: integer + description: 'If set to 1 confidential data will not be hidden. This means that the response will contain the key and additional data for secrets. If a master key is not set or an external KMS is used, the data returned are enough to get the secrets in cleartext. Ignored if the manage_system permission is not granted.' requestBody: required: true content: @@ -1613,6 +1619,12 @@ paths: summary: Find folders by name description: Returns the folder with the given name if it exists. operationId: get_folder_by_name + parameters: + - in: query + name: confidential_data + schema: + type: integer + description: 'If set to 1 confidential data will not be hidden. This means that the response will contain the key and additional data for secrets. If a master key is not set or an external KMS is used, the data returned are enough to get the secrets in cleartext. Ignored if the manage_system permission is not granted.' responses: '200': description: successful operation @@ -1751,6 +1763,12 @@ paths: summary: Add group operationId: add_group description: Adds a new group + parameters: + - in: query + name: confidential_data + schema: + type: integer + description: 'If set to 1 confidential data will not be hidden. This means that the response will contain the key and additional data for secrets. If a master key is not set or an external KMS is used, the data returned are enough to get the secrets in cleartext. Ignored if the manage_system permission is not granted.' requestBody: required: true content: @@ -1793,6 +1811,12 @@ paths: summary: Find groups by name description: Returns the group with the given name if it exists. operationId: get_group_by_name + parameters: + - in: query + name: confidential_data + schema: + type: integer + description: 'If set to 1 confidential data will not be hidden. This means that the response will contain the key and additional data for secrets. If a master key is not set or an external KMS is used, the data returned are enough to get the secrets in cleartext. Ignored if the manage_system permission is not granted.' responses: '200': description: successful operation @@ -2111,6 +2135,12 @@ paths: summary: Add event action operationId: add_event_action description: Adds a new event actions + parameters: + - in: query + name: confidential_data + schema: + type: integer + description: 'If set to 1 confidential data will not be hidden. This means that the response will contain the key and additional data for secrets. If a master key is not set or an external KMS is used, the data returned are enough to get the secrets in cleartext. Ignored if the manage_system permission is not granted.' requestBody: required: true content: @@ -2153,6 +2183,12 @@ paths: summary: Find event actions by name description: Returns the event action with the given name if it exists. operationId: get_event_action_by_name + parameters: + - in: query + name: confidential_data + schema: + type: integer + description: 'If set to 1 confidential data will not be hidden. This means that the response will contain the key and additional data for secrets. If a master key is not set or an external KMS is used, the data returned are enough to get the secrets in cleartext. Ignored if the manage_system permission is not granted.' responses: '200': description: successful operation @@ -2291,6 +2327,12 @@ paths: summary: Add event rule operationId: add_event_rule description: Adds a new event rule + parameters: + - in: query + name: confidential_data + schema: + type: integer + description: 'If set to 1 confidential data will not be hidden. This means that the response will contain the key and additional data for secrets. If a master key is not set or an external KMS is used, the data returned are enough to get the secrets in cleartext. Ignored if the manage_system permission is not granted.' requestBody: required: true content: @@ -2333,6 +2375,12 @@ paths: summary: Find event rules by name description: Returns the event rule with the given name if it exists. operationId: get_event_rile_by_name + parameters: + - in: query + name: confidential_data + schema: + type: integer + description: 'If set to 1 confidential data will not be hidden. This means that the response will contain the key and additional data for secrets. If a master key is not set or an external KMS is used, the data returned are enough to get the secrets in cleartext. Ignored if the manage_system permission is not granted.' responses: '200': description: successful operation @@ -3303,6 +3351,12 @@ paths: summary: Add user description: 'Adds a new user.Recovery codes and TOTP configuration cannot be set using this API: each user must use the specific APIs' operationId: add_user + parameters: + - in: query + name: confidential_data + schema: + type: integer + description: 'If set to 1 confidential data will not be hidden. This means that the response will contain the hash of the password and the key and additional data for secrets. If a master key is not set or an external KMS is used, the data returned are enough to get the secrets in cleartext. Ignored if the manage_system permission is not granted.' requestBody: required: true content: @@ -3345,6 +3399,12 @@ paths: summary: Find users by username description: Returns the user with the given username if it exists. For security reasons the hashed password is omitted in the response operationId: get_user_by_username + parameters: + - in: query + name: confidential_data + schema: + type: integer + description: 'If set to 1 confidential data will not be hidden. This means that the response will contain the hash of the password and the key and additional data for secrets. If a master key is not set or an external KMS is used, the data returned are enough to get the secrets in cleartext. Ignored if the manage_system permission is not granted.' responses: '200': description: successful operation @@ -3609,6 +3669,15 @@ paths: indent: * `0` no indentation. This is the default * `1` format the output JSON + - in: query + name: scopes + schema: + type: array + items: + $ref: '#/components/schemas/DumpDataScopes' + description: 'You can limit the dump contents to the specified scopes. Empty or missing means any supported scope. Scopes must be specified comma separated' + explode: false + required: false responses: '200': description: successful operation @@ -5056,6 +5125,20 @@ components: - LDAPUser - OSUser description: This is an hint for authentication plugins. It is ignored when using SFTPGo internal authentication + DumpDataScopes: + type: string + enum: + - users + - folders + - groups + - admins + - api_keys + - shares + - actions + - rules + - roles + - ip_lists + - configs FsEventStatus: type: integer enum: