diff --git a/dataprovider/bolt.go b/dataprovider/bolt.go index 34673190..1d21e26d 100644 --- a/dataprovider/bolt.go +++ b/dataprovider/bolt.go @@ -1160,10 +1160,18 @@ func (p *BoltProvider) addShare(share *Share) error { return err } share.ID = int64(id) - share.CreatedAt = util.GetTimeAsMsSinceEpoch(time.Now()) - share.UpdatedAt = util.GetTimeAsMsSinceEpoch(time.Now()) - share.LastUseAt = 0 - share.UsedTokens = 0 + if !share.IsRestore { + share.CreatedAt = util.GetTimeAsMsSinceEpoch(time.Now()) + share.UpdatedAt = share.CreatedAt + share.LastUseAt = 0 + share.UsedTokens = 0 + } + if share.CreatedAt == 0 { + share.CreatedAt = util.GetTimeAsMsSinceEpoch(time.Now()) + } + if share.UpdatedAt == 0 { + share.UpdatedAt = share.CreatedAt + } if err := p.userExistsInternal(tx, share.Username); err != nil { return util.NewValidationError(fmt.Sprintf("related user %#v does not exists", share.Username)) } @@ -1200,10 +1208,18 @@ func (p *BoltProvider) updateShare(share *Share) error { share.ID = oldObject.ID share.ShareID = oldObject.ShareID - share.UsedTokens = oldObject.UsedTokens - share.CreatedAt = oldObject.CreatedAt - share.LastUseAt = oldObject.LastUseAt - share.UpdatedAt = util.GetTimeAsMsSinceEpoch(time.Now()) + if !share.IsRestore { + share.UsedTokens = oldObject.UsedTokens + share.CreatedAt = oldObject.CreatedAt + share.LastUseAt = oldObject.LastUseAt + share.UpdatedAt = util.GetTimeAsMsSinceEpoch(time.Now()) + } + if share.CreatedAt == 0 { + share.CreatedAt = util.GetTimeAsMsSinceEpoch(time.Now()) + } + if share.UpdatedAt == 0 { + share.UpdatedAt = share.CreatedAt + } if err := p.userExistsInternal(tx, share.Username); err != nil { return util.NewValidationError(fmt.Sprintf("related user %#v does not exists", share.Username)) } diff --git a/dataprovider/memory.go b/dataprovider/memory.go index bcf31128..b8e4348f 100644 --- a/dataprovider/memory.go +++ b/dataprovider/memory.go @@ -1103,10 +1103,18 @@ func (p *MemoryProvider) addShare(share *Share) error { if _, err := p.userExistsInternal(share.Username); err != nil { return util.NewValidationError(fmt.Sprintf("related user %#v does not exists", share.Username)) } - share.CreatedAt = util.GetTimeAsMsSinceEpoch(time.Now()) - share.UpdatedAt = util.GetTimeAsMsSinceEpoch(time.Now()) - share.LastUseAt = 0 - share.UsedTokens = 0 + if !share.IsRestore { + share.CreatedAt = util.GetTimeAsMsSinceEpoch(time.Now()) + share.UpdatedAt = share.CreatedAt + share.LastUseAt = 0 + share.UsedTokens = 0 + } + if share.CreatedAt == 0 { + share.CreatedAt = util.GetTimeAsMsSinceEpoch(time.Now()) + } + if share.UpdatedAt == 0 { + share.UpdatedAt = share.CreatedAt + } p.dbHandle.shares[share.ShareID] = share.getACopy() p.dbHandle.sharesIDs = append(p.dbHandle.sharesIDs, share.ShareID) sort.Strings(p.dbHandle.sharesIDs) @@ -1133,10 +1141,18 @@ func (p *MemoryProvider) updateShare(share *Share) error { } share.ID = s.ID share.ShareID = s.ShareID - share.UsedTokens = s.UsedTokens - share.CreatedAt = s.CreatedAt - share.LastUseAt = s.LastUseAt - share.UpdatedAt = util.GetTimeAsMsSinceEpoch(time.Now()) + if !share.IsRestore { + share.UsedTokens = s.UsedTokens + share.CreatedAt = s.CreatedAt + share.LastUseAt = s.LastUseAt + share.UpdatedAt = util.GetTimeAsMsSinceEpoch(time.Now()) + } + if share.CreatedAt == 0 { + share.CreatedAt = util.GetTimeAsMsSinceEpoch(time.Now()) + } + if share.UpdatedAt == 0 { + share.UpdatedAt = share.CreatedAt + } p.dbHandle.shares[share.ShareID] = share.getACopy() return nil } @@ -1346,6 +1362,7 @@ func (p *MemoryProvider) restoreShares(dump *BackupData) error { for _, share := range dump.Shares { s, err := p.shareExists(share.ShareID, "") share := share // pin + share.IsRestore = true if err == nil { share.ID = s.ID err = UpdateShare(&share, ActionExecutorSystem, "") diff --git a/dataprovider/share.go b/dataprovider/share.go index ec1d10b9..1a190d98 100644 --- a/dataprovider/share.go +++ b/dataprovider/share.go @@ -54,6 +54,10 @@ type Share struct { UsedTokens int `json:"used_tokens,omitempty"` // Limit the share availability to these IPs/CIDR networks AllowFrom []string `json:"allow_from,omitempty"` + // set for restores, we don't have to validate the expiration date + // otherwise we fail to restore existing shares and we have to insert + // all the previous values with no modifications + IsRestore bool `json:"-"` } // GetScopeAsString returns the share's scope as string. @@ -210,7 +214,7 @@ func (s *Share) validate() error { return err } if s.ExpiresAt > 0 { - if s.ExpiresAt < util.GetTimeAsMsSinceEpoch(time.Now()) { + if !s.IsRestore && s.ExpiresAt < util.GetTimeAsMsSinceEpoch(time.Now()) { return util.NewValidationError("expiration must be in the future") } } else { diff --git a/dataprovider/sqlcommon.go b/dataprovider/sqlcommon.go index 918e6311..dfed368b 100644 --- a/dataprovider/sqlcommon.go +++ b/dataprovider/sqlcommon.go @@ -93,9 +93,23 @@ func sqlCommonAddShare(share *Share, dbHandle *sql.DB) error { } defer stmt.Close() + usedTokens := 0 + createdAt := util.GetTimeAsMsSinceEpoch(time.Now()) + updatedAt := createdAt + lastUseAt := int64(0) + if share.IsRestore { + usedTokens = share.UsedTokens + if share.CreatedAt > 0 { + createdAt = share.CreatedAt + } + if share.UpdatedAt > 0 { + updatedAt = share.UpdatedAt + } + lastUseAt = share.LastUseAt + } _, err = stmt.ExecContext(ctx, share.ShareID, share.Name, share.Description, share.Scope, - string(paths), util.GetTimeAsMsSinceEpoch(time.Now()), util.GetTimeAsMsSinceEpoch(time.Now()), - share.LastUseAt, share.ExpiresAt, share.Password, share.MaxTokens, allowFrom, user.ID) + string(paths), createdAt, updatedAt, lastUseAt, share.ExpiresAt, share.Password, + share.MaxTokens, usedTokens, allowFrom, user.ID) return err } @@ -125,7 +139,12 @@ func sqlCommonUpdateShare(share *Share, dbHandle *sql.DB) error { ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout) defer cancel() - q := getUpdateShareQuery() + var q string + if share.IsRestore { + q = getUpdateShareRestoreQuery() + } else { + q = getUpdateShareQuery() + } stmt, err := dbHandle.PrepareContext(ctx, q) if err != nil { providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err) @@ -133,9 +152,21 @@ func sqlCommonUpdateShare(share *Share, dbHandle *sql.DB) error { } defer stmt.Close() - _, err = stmt.ExecContext(ctx, share.Name, share.Description, share.Scope, string(paths), - util.GetTimeAsMsSinceEpoch(time.Now()), share.ExpiresAt, share.Password, share.MaxTokens, - allowFrom, user.ID, share.ShareID) + if share.IsRestore { + if share.CreatedAt == 0 { + share.CreatedAt = util.GetTimeAsMsSinceEpoch(time.Now()) + } + if share.UpdatedAt == 0 { + share.UpdatedAt = share.CreatedAt + } + _, err = stmt.ExecContext(ctx, share.Name, share.Description, share.Scope, string(paths), + share.CreatedAt, share.UpdatedAt, share.LastUseAt, share.ExpiresAt, share.Password, share.MaxTokens, + share.UsedTokens, allowFrom, user.ID, share.ShareID) + } else { + _, err = stmt.ExecContext(ctx, share.Name, share.Description, share.Scope, string(paths), + util.GetTimeAsMsSinceEpoch(time.Now()), share.ExpiresAt, share.Password, share.MaxTokens, + allowFrom, user.ID, share.ShareID) + } return err } diff --git a/dataprovider/sqlqueries.go b/dataprovider/sqlqueries.go index ce186352..ad74f5a4 100644 --- a/dataprovider/sqlqueries.go +++ b/dataprovider/sqlqueries.go @@ -82,11 +82,19 @@ func getDumpSharesQuery() string { func getAddShareQuery() string { return fmt.Sprintf(`INSERT INTO %v (share_id,name,description,scope,paths,created_at,updated_at,last_use_at, - expires_at,password,max_tokens,used_tokens,allow_from,user_id) VALUES (%v,%v,%v,%v,%v,%v,%v,%v,%v,%v,%v,0,%v,%v)`, + expires_at,password,max_tokens,used_tokens,allow_from,user_id) VALUES (%v,%v,%v,%v,%v,%v,%v,%v,%v,%v,%v,%v,%v,%v)`, sqlTableShares, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3], sqlPlaceholders[4], sqlPlaceholders[5], sqlPlaceholders[6], sqlPlaceholders[7], sqlPlaceholders[8], sqlPlaceholders[9], sqlPlaceholders[10], sqlPlaceholders[11], - sqlPlaceholders[12]) + sqlPlaceholders[12], sqlPlaceholders[13]) +} + +func getUpdateShareRestoreQuery() string { + return fmt.Sprintf(`UPDATE %v SET name=%v,description=%v,scope=%v,paths=%v,created_at=%v,updated_at=%v, + last_use_at=%v,expires_at=%v,password=%v,max_tokens=%v,used_tokens=%v,allow_from=%v,user_id=%v WHERE share_id = %v`, sqlTableShares, + sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3], sqlPlaceholders[4], + sqlPlaceholders[5], sqlPlaceholders[6], sqlPlaceholders[7], sqlPlaceholders[8], sqlPlaceholders[9], + sqlPlaceholders[10], sqlPlaceholders[11], sqlPlaceholders[12], sqlPlaceholders[13]) } func getUpdateShareQuery() string { diff --git a/go.mod b/go.mod index bbb61732..6211a64a 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/Azure/azure-storage-blob-go v0.14.0 github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962 github.com/alexedwards/argon2id v0.0.0-20210511081203-7d35d68092b8 - github.com/aws/aws-sdk-go v1.42.12 + github.com/aws/aws-sdk-go v1.42.13 github.com/cockroachdb/cockroach-go/v2 v2.2.4 github.com/eikenb/pipeat v0.0.0-20210603033007-44fc3ffce52b github.com/fclairamb/ftpserverlib v0.16.0 diff --git a/go.sum b/go.sum index 3a64bc53..d08e5c11 100644 --- a/go.sum +++ b/go.sum @@ -137,8 +137,8 @@ github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZo github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.38.68/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.40.34/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= -github.com/aws/aws-sdk-go v1.42.12 h1:zVrAgi3/HuMPygZknc+f2KAHcn+Zuq767857hnHBMPA= -github.com/aws/aws-sdk-go v1.42.12/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= +github.com/aws/aws-sdk-go v1.42.13 h1:+Nx87T+Bjiq2XybxK6vI98cTEBPLE/hILuZyEenlyEg= +github.com/aws/aws-sdk-go v1.42.13/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/aws/aws-sdk-go-v2 v1.7.0/go.mod h1:tb9wi5s61kTDA5qCkcDbt3KRVV74GGslQkl/DRdX/P4= github.com/aws/aws-sdk-go-v2 v1.9.0/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= github.com/aws/aws-sdk-go-v2/config v1.7.0/go.mod h1:w9+nMZ7soXCe5nT46Ri354SNhXDQ6v+V5wqDjnZE+GY= diff --git a/httpd/api_maintenance.go b/httpd/api_maintenance.go index 17fe844b..8dcd3a47 100644 --- a/httpd/api_maintenance.go +++ b/httpd/api_maintenance.go @@ -254,6 +254,7 @@ func RestoreShares(shares []dataprovider.Share, inputFile string, mode int, exec ) error { for _, share := range shares { share := share // pin + share.IsRestore = true s, err := dataprovider.ShareExists(share.ShareID, "") if err == nil { if mode == 1 { diff --git a/httpd/httpd_test.go b/httpd/httpd_test.go index f9f39566..422d919d 100644 --- a/httpd/httpd_test.go +++ b/httpd/httpd_test.go @@ -27,6 +27,7 @@ import ( "github.com/go-chi/render" _ "github.com/go-sql-driver/mysql" _ "github.com/lib/pq" + "github.com/lithammer/shortuuid/v3" _ "github.com/mattn/go-sqlite3" "github.com/mhale/smtpd" "github.com/pquerna/otp" @@ -4270,6 +4271,54 @@ func TestDefenderAPIErrors(t *testing.T) { require.NoError(t, err) } +func TestRestoreShares(t *testing.T) { + // shares should be restored preserving the UsedTokens, CreatedAt, LastUseAt, UpdatedAt, + // and ExpiresAt, so an expired share can be restored while we cannot create an already + // expired share + user, _, err := httpdtest.AddUser(getTestUser(), http.StatusCreated) + assert.NoError(t, err) + share := dataprovider.Share{ + ShareID: shortuuid.New(), + Name: "share name", + Description: "share description", + Scope: dataprovider.ShareScopeRead, + Paths: []string{"/"}, + Username: user.Username, + CreatedAt: util.GetTimeAsMsSinceEpoch(time.Now().Add(-144 * time.Hour)), + UpdatedAt: util.GetTimeAsMsSinceEpoch(time.Now().Add(-96 * time.Hour)), + LastUseAt: util.GetTimeAsMsSinceEpoch(time.Now().Add(-64 * time.Hour)), + ExpiresAt: util.GetTimeAsMsSinceEpoch(time.Now().Add(-48 * time.Hour)), + MaxTokens: 10, + UsedTokens: 8, + AllowFrom: []string{"127.0.0.0/8"}, + } + backupData := dataprovider.BackupData{} + backupData.Shares = append(backupData.Shares, share) + backupContent, err := json.Marshal(backupData) + assert.NoError(t, err) + _, _, err = httpdtest.LoaddataFromPostBody(backupContent, "0", "0", http.StatusOK) + assert.NoError(t, err) + shareGet, err := dataprovider.ShareExists(share.ShareID, user.Username) + assert.NoError(t, err) + assert.Equal(t, share, shareGet) + + share.CreatedAt = util.GetTimeAsMsSinceEpoch(time.Now().Add(-142 * time.Hour)) + share.UpdatedAt = util.GetTimeAsMsSinceEpoch(time.Now().Add(-92 * time.Hour)) + share.LastUseAt = util.GetTimeAsMsSinceEpoch(time.Now().Add(-62 * time.Hour)) + share.UsedTokens = 6 + backupData.Shares = []dataprovider.Share{share} + backupContent, err = json.Marshal(backupData) + assert.NoError(t, err) + _, _, err = httpdtest.LoaddataFromPostBody(backupContent, "0", "0", http.StatusOK) + assert.NoError(t, err) + shareGet, err = dataprovider.ShareExists(share.ShareID, user.Username) + assert.NoError(t, err) + assert.Equal(t, share, shareGet) + + _, err = httpdtest.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) +} + func TestLoaddataFromPostBody(t *testing.T) { mappedPath := filepath.Join(os.TempDir(), "restored_folder") folderName := filepath.Base(mappedPath) diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index 64ea2d15..30cfcfb2 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -2721,14 +2721,14 @@ paths: - 2 description: | Mode: - * `0` New users/admins/API keys are added, existing ones are updated. This is the default - * `1` New users/admins/API keys are added, existing ones are not modified - * `2` New users/admins/API keys are added, existing ones are updated and connected users are disconnected and so forced to use the new configuration + * `0` New objects are added, existing ones are updated. This is the default + * `1` New objects are added, existing ones are not modified + * `2` New objects are added, existing ones are updated and connected users are disconnected and so forced to use the new configuration get: tags: - maintenance summary: Load data from path - description: 'Restores SFTPGo data from a JSON backup file on the server. Users, folders and admins will be restored one by one and the restore is stopped if a user/folder/admin cannot be added or updated, so it could happen a partial restore' + description: 'Restores SFTPGo data from a JSON backup file on the server. Objects will be restored one by one and the restore is stopped if a object cannot be added or updated, so it could happen a partial restore' operationId: loaddata_from_file parameters: - in: query @@ -2760,7 +2760,7 @@ paths: tags: - maintenance summary: Load data - description: 'Restores SFTPGo data from a JSON backup. Users, folders and admins will be restored one by one and the restore is stopped if a user/folder/admin cannot be added or updated, so it could happen a partial restore' + description: 'Restores SFTPGo data from a JSON backup. Objects will be restored one by one and the restore is stopped if a object cannot be added or updated, so it could happen a partial restore' operationId: loaddata_from_request_body requestBody: required: true