loaddata: improve shares restore

usage and timestamps are now preserved
2021-11-27 11:12:51 +01:00 · 2021-11-27 11:12:51 +01:00 · 015aa36c56
commit 015aa36c56
parent f2480ce5c9
10 changed files with 159 additions and 33 deletions
--- a/dataprovider/bolt.go
+++ b/dataprovider/bolt.go
@ -1160,10 +1160,18 @@ func (p *BoltProvider) addShare(share *Share) error {
 			return err
 		}
 		share.ID = int64(id)
-		share.CreatedAt = util.GetTimeAsMsSinceEpoch(time.Now())
-		share.UpdatedAt = util.GetTimeAsMsSinceEpoch(time.Now())
-		share.LastUseAt = 0
-		share.UsedTokens = 0
+		if !share.IsRestore {
+			share.CreatedAt = util.GetTimeAsMsSinceEpoch(time.Now())
+			share.UpdatedAt = share.CreatedAt
+			share.LastUseAt = 0
+			share.UsedTokens = 0
+		}
+		if share.CreatedAt == 0 {
+			share.CreatedAt = util.GetTimeAsMsSinceEpoch(time.Now())
+		}
+		if share.UpdatedAt == 0 {
+			share.UpdatedAt = share.CreatedAt
+		}
 		if err := p.userExistsInternal(tx, share.Username); err != nil {
 			return util.NewValidationError(fmt.Sprintf("related user %#v does not exists", share.Username))
 		}
@ -1200,10 +1208,18 @@ func (p *BoltProvider) updateShare(share *Share) error {

 		share.ID = oldObject.ID
 		share.ShareID = oldObject.ShareID
-		share.UsedTokens = oldObject.UsedTokens
-		share.CreatedAt = oldObject.CreatedAt
-		share.LastUseAt = oldObject.LastUseAt
-		share.UpdatedAt = util.GetTimeAsMsSinceEpoch(time.Now())
+		if !share.IsRestore {
+			share.UsedTokens = oldObject.UsedTokens
+			share.CreatedAt = oldObject.CreatedAt
+			share.LastUseAt = oldObject.LastUseAt
+			share.UpdatedAt = util.GetTimeAsMsSinceEpoch(time.Now())
+		}
+		if share.CreatedAt == 0 {
+			share.CreatedAt = util.GetTimeAsMsSinceEpoch(time.Now())
+		}
+		if share.UpdatedAt == 0 {
+			share.UpdatedAt = share.CreatedAt
+		}
 		if err := p.userExistsInternal(tx, share.Username); err != nil {
 			return util.NewValidationError(fmt.Sprintf("related user %#v does not exists", share.Username))
 		}
--- a/dataprovider/memory.go
+++ b/dataprovider/memory.go
@ -1103,10 +1103,18 @@ func (p *MemoryProvider) addShare(share *Share) error {
 	if _, err := p.userExistsInternal(share.Username); err != nil {
 		return util.NewValidationError(fmt.Sprintf("related user %#v does not exists", share.Username))
 	}
-	share.CreatedAt = util.GetTimeAsMsSinceEpoch(time.Now())
-	share.UpdatedAt = util.GetTimeAsMsSinceEpoch(time.Now())
-	share.LastUseAt = 0
-	share.UsedTokens = 0
+	if !share.IsRestore {
+		share.CreatedAt = util.GetTimeAsMsSinceEpoch(time.Now())
+		share.UpdatedAt = share.CreatedAt
+		share.LastUseAt = 0
+		share.UsedTokens = 0
+	}
+	if share.CreatedAt == 0 {
+		share.CreatedAt = util.GetTimeAsMsSinceEpoch(time.Now())
+	}
+	if share.UpdatedAt == 0 {
+		share.UpdatedAt = share.CreatedAt
+	}
 	p.dbHandle.shares[share.ShareID] = share.getACopy()
 	p.dbHandle.sharesIDs = append(p.dbHandle.sharesIDs, share.ShareID)
 	sort.Strings(p.dbHandle.sharesIDs)
@ -1133,10 +1141,18 @@ func (p *MemoryProvider) updateShare(share *Share) error {
 	}
 	share.ID = s.ID
 	share.ShareID = s.ShareID
-	share.UsedTokens = s.UsedTokens
-	share.CreatedAt = s.CreatedAt
-	share.LastUseAt = s.LastUseAt
-	share.UpdatedAt = util.GetTimeAsMsSinceEpoch(time.Now())
+	if !share.IsRestore {
+		share.UsedTokens = s.UsedTokens
+		share.CreatedAt = s.CreatedAt
+		share.LastUseAt = s.LastUseAt
+		share.UpdatedAt = util.GetTimeAsMsSinceEpoch(time.Now())
+	}
+	if share.CreatedAt == 0 {
+		share.CreatedAt = util.GetTimeAsMsSinceEpoch(time.Now())
+	}
+	if share.UpdatedAt == 0 {
+		share.UpdatedAt = share.CreatedAt
+	}
 	p.dbHandle.shares[share.ShareID] = share.getACopy()
 	return nil
 }
@ -1346,6 +1362,7 @@ func (p *MemoryProvider) restoreShares(dump *BackupData) error {
 	for _, share := range dump.Shares {
 		s, err := p.shareExists(share.ShareID, "")
 		share := share // pin
+		share.IsRestore = true
 		if err == nil {
 			share.ID = s.ID
 			err = UpdateShare(&share, ActionExecutorSystem, "")
--- a/dataprovider/share.go
+++ b/dataprovider/share.go
@ -54,6 +54,10 @@ type Share struct {
 	UsedTokens int `json:"used_tokens,omitempty"`
 	// Limit the share availability to these IPs/CIDR networks
 	AllowFrom []string `json:"allow_from,omitempty"`
+	// set for restores, we don't have to validate the expiration date
+	// otherwise we fail to restore existing shares and we have to insert
+	// all the previous values with no modifications
+	IsRestore bool `json:"-"`
 }

 // GetScopeAsString returns the share's scope as string.
@ -210,7 +214,7 @@ func (s *Share) validate() error {
 		return err
 	}
 	if s.ExpiresAt > 0 {
-		if s.ExpiresAt < util.GetTimeAsMsSinceEpoch(time.Now()) {
+		if !s.IsRestore && s.ExpiresAt < util.GetTimeAsMsSinceEpoch(time.Now()) {
 			return util.NewValidationError("expiration must be in the future")
 		}
 	} else {
--- a/dataprovider/sqlcommon.go
+++ b/dataprovider/sqlcommon.go
@ -93,9 +93,23 @@ func sqlCommonAddShare(share *Share, dbHandle *sql.DB) error {
 	}
 	defer stmt.Close()

+	usedTokens := 0
+	createdAt := util.GetTimeAsMsSinceEpoch(time.Now())
+	updatedAt := createdAt
+	lastUseAt := int64(0)
+	if share.IsRestore {
+		usedTokens = share.UsedTokens
+		if share.CreatedAt > 0 {
+			createdAt = share.CreatedAt
+		}
+		if share.UpdatedAt > 0 {
+			updatedAt = share.UpdatedAt
+		}
+		lastUseAt = share.LastUseAt
+	}
 	_, err = stmt.ExecContext(ctx, share.ShareID, share.Name, share.Description, share.Scope,
-		string(paths), util.GetTimeAsMsSinceEpoch(time.Now()), util.GetTimeAsMsSinceEpoch(time.Now()),
-		share.LastUseAt, share.ExpiresAt, share.Password, share.MaxTokens, allowFrom, user.ID)
+		string(paths), createdAt, updatedAt, lastUseAt, share.ExpiresAt, share.Password,
+		share.MaxTokens, usedTokens, allowFrom, user.ID)
 	return err
 }

@ -125,7 +139,12 @@ func sqlCommonUpdateShare(share *Share, dbHandle *sql.DB) error {

 	ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout)
 	defer cancel()
-	q := getUpdateShareQuery()
+	var q string
+	if share.IsRestore {
+		q = getUpdateShareRestoreQuery()
+	} else {
+		q = getUpdateShareQuery()
+	}
 	stmt, err := dbHandle.PrepareContext(ctx, q)
 	if err != nil {
 		providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
@ -133,9 +152,21 @@ func sqlCommonUpdateShare(share *Share, dbHandle *sql.DB) error {
 	}
 	defer stmt.Close()

-	_, err = stmt.ExecContext(ctx, share.Name, share.Description, share.Scope, string(paths),
-		util.GetTimeAsMsSinceEpoch(time.Now()), share.ExpiresAt, share.Password, share.MaxTokens,
-		allowFrom, user.ID, share.ShareID)
+	if share.IsRestore {
+		if share.CreatedAt == 0 {
+			share.CreatedAt = util.GetTimeAsMsSinceEpoch(time.Now())
+		}
+		if share.UpdatedAt == 0 {
+			share.UpdatedAt = share.CreatedAt
+		}
+		_, err = stmt.ExecContext(ctx, share.Name, share.Description, share.Scope, string(paths),
+			share.CreatedAt, share.UpdatedAt, share.LastUseAt, share.ExpiresAt, share.Password, share.MaxTokens,
+			share.UsedTokens, allowFrom, user.ID, share.ShareID)
+	} else {
+		_, err = stmt.ExecContext(ctx, share.Name, share.Description, share.Scope, string(paths),
+			util.GetTimeAsMsSinceEpoch(time.Now()), share.ExpiresAt, share.Password, share.MaxTokens,
+			allowFrom, user.ID, share.ShareID)
+	}
 	return err
 }

--- a/dataprovider/sqlqueries.go
+++ b/dataprovider/sqlqueries.go
@ -82,11 +82,19 @@ func getDumpSharesQuery() string {

 func getAddShareQuery() string {
 	return fmt.Sprintf(`INSERT INTO %v (share_id,name,description,scope,paths,created_at,updated_at,last_use_at,
-		expires_at,password,max_tokens,used_tokens,allow_from,user_id) VALUES (%v,%v,%v,%v,%v,%v,%v,%v,%v,%v,%v,0,%v,%v)`,
+		expires_at,password,max_tokens,used_tokens,allow_from,user_id) VALUES (%v,%v,%v,%v,%v,%v,%v,%v,%v,%v,%v,%v,%v,%v)`,
 		sqlTableShares, sqlPlaceholders[0], sqlPlaceholders[1],
 		sqlPlaceholders[2], sqlPlaceholders[3], sqlPlaceholders[4], sqlPlaceholders[5], sqlPlaceholders[6],
 		sqlPlaceholders[7], sqlPlaceholders[8], sqlPlaceholders[9], sqlPlaceholders[10], sqlPlaceholders[11],
-		sqlPlaceholders[12])
+		sqlPlaceholders[12], sqlPlaceholders[13])
+}
+
+func getUpdateShareRestoreQuery() string {
+	return fmt.Sprintf(`UPDATE %v SET name=%v,description=%v,scope=%v,paths=%v,created_at=%v,updated_at=%v,
+		last_use_at=%v,expires_at=%v,password=%v,max_tokens=%v,used_tokens=%v,allow_from=%v,user_id=%v WHERE share_id = %v`, sqlTableShares,
+		sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3], sqlPlaceholders[4],
+		sqlPlaceholders[5], sqlPlaceholders[6], sqlPlaceholders[7], sqlPlaceholders[8], sqlPlaceholders[9],
+		sqlPlaceholders[10], sqlPlaceholders[11], sqlPlaceholders[12], sqlPlaceholders[13])
 }

 func getUpdateShareQuery() string {
--- a/go.mod
+++ b/go.mod
@ -7,7 +7,7 @@ require (
 	github.com/Azure/azure-storage-blob-go v0.14.0
 	github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962
 	github.com/alexedwards/argon2id v0.0.0-20210511081203-7d35d68092b8
-	github.com/aws/aws-sdk-go v1.42.12
+	github.com/aws/aws-sdk-go v1.42.13
 	github.com/cockroachdb/cockroach-go/v2 v2.2.4
 	github.com/eikenb/pipeat v0.0.0-20210603033007-44fc3ffce52b
 	github.com/fclairamb/ftpserverlib v0.16.0
--- a/go.sum
+++ b/go.sum
@ -137,8 +137,8 @@ github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZo
 github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
 github.com/aws/aws-sdk-go v1.38.68/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
 github.com/aws/aws-sdk-go v1.40.34/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
-github.com/aws/aws-sdk-go v1.42.12 h1:zVrAgi3/HuMPygZknc+f2KAHcn+Zuq767857hnHBMPA=
-github.com/aws/aws-sdk-go v1.42.12/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
+github.com/aws/aws-sdk-go v1.42.13 h1:+Nx87T+Bjiq2XybxK6vI98cTEBPLE/hILuZyEenlyEg=
+github.com/aws/aws-sdk-go v1.42.13/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
 github.com/aws/aws-sdk-go-v2 v1.7.0/go.mod h1:tb9wi5s61kTDA5qCkcDbt3KRVV74GGslQkl/DRdX/P4=
 github.com/aws/aws-sdk-go-v2 v1.9.0/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4=
 github.com/aws/aws-sdk-go-v2/config v1.7.0/go.mod h1:w9+nMZ7soXCe5nT46Ri354SNhXDQ6v+V5wqDjnZE+GY=
--- a/httpd/api_maintenance.go
+++ b/httpd/api_maintenance.go
@ -254,6 +254,7 @@ func RestoreShares(shares []dataprovider.Share, inputFile string, mode int, exec
 ) error {
 	for _, share := range shares {
 		share := share // pin
+		share.IsRestore = true
 		s, err := dataprovider.ShareExists(share.ShareID, "")
 		if err == nil {
 			if mode == 1 {
--- a/httpd/httpd_test.go
+++ b/httpd/httpd_test.go
@ -27,6 +27,7 @@ import (
 	"github.com/go-chi/render"
 	_ "github.com/go-sql-driver/mysql"
 	_ "github.com/lib/pq"
+	"github.com/lithammer/shortuuid/v3"
 	_ "github.com/mattn/go-sqlite3"
 	"github.com/mhale/smtpd"
 	"github.com/pquerna/otp"
@ -4270,6 +4271,54 @@ func TestDefenderAPIErrors(t *testing.T) {
 	require.NoError(t, err)
 }

+func TestRestoreShares(t *testing.T) {
+	// shares should be restored preserving the UsedTokens, CreatedAt, LastUseAt, UpdatedAt,
+	// and ExpiresAt, so an expired share can be restored while we cannot create an already
+	// expired share
+	user, _, err := httpdtest.AddUser(getTestUser(), http.StatusCreated)
+	assert.NoError(t, err)
+	share := dataprovider.Share{
+		ShareID:     shortuuid.New(),
+		Name:        "share name",
+		Description: "share description",
+		Scope:       dataprovider.ShareScopeRead,
+		Paths:       []string{"/"},
+		Username:    user.Username,
+		CreatedAt:   util.GetTimeAsMsSinceEpoch(time.Now().Add(-144 * time.Hour)),
+		UpdatedAt:   util.GetTimeAsMsSinceEpoch(time.Now().Add(-96 * time.Hour)),
+		LastUseAt:   util.GetTimeAsMsSinceEpoch(time.Now().Add(-64 * time.Hour)),
+		ExpiresAt:   util.GetTimeAsMsSinceEpoch(time.Now().Add(-48 * time.Hour)),
+		MaxTokens:   10,
+		UsedTokens:  8,
+		AllowFrom:   []string{"127.0.0.0/8"},
+	}
+	backupData := dataprovider.BackupData{}
+	backupData.Shares = append(backupData.Shares, share)
+	backupContent, err := json.Marshal(backupData)
+	assert.NoError(t, err)
+	_, _, err = httpdtest.LoaddataFromPostBody(backupContent, "0", "0", http.StatusOK)
+	assert.NoError(t, err)
+	shareGet, err := dataprovider.ShareExists(share.ShareID, user.Username)
+	assert.NoError(t, err)
+	assert.Equal(t, share, shareGet)
+
+	share.CreatedAt = util.GetTimeAsMsSinceEpoch(time.Now().Add(-142 * time.Hour))
+	share.UpdatedAt = util.GetTimeAsMsSinceEpoch(time.Now().Add(-92 * time.Hour))
+	share.LastUseAt = util.GetTimeAsMsSinceEpoch(time.Now().Add(-62 * time.Hour))
+	share.UsedTokens = 6
+	backupData.Shares = []dataprovider.Share{share}
+	backupContent, err = json.Marshal(backupData)
+	assert.NoError(t, err)
+	_, _, err = httpdtest.LoaddataFromPostBody(backupContent, "0", "0", http.StatusOK)
+	assert.NoError(t, err)
+	shareGet, err = dataprovider.ShareExists(share.ShareID, user.Username)
+	assert.NoError(t, err)
+	assert.Equal(t, share, shareGet)
+
+	_, err = httpdtest.RemoveUser(user, http.StatusOK)
+	assert.NoError(t, err)
+}
+
 func TestLoaddataFromPostBody(t *testing.T) {
 	mappedPath := filepath.Join(os.TempDir(), "restored_folder")
 	folderName := filepath.Base(mappedPath)
--- a/openapi/openapi.yaml
+++ b/openapi/openapi.yaml
@ -2721,14 +2721,14 @@ paths:
            - 2
        description: |
          Mode:
-            * `0` New users/admins/API keys are added, existing ones are updated. This is the default
-            * `1` New users/admins/API keys are added, existing ones are not modified
-            * `2` New users/admins/API keys are added, existing ones are updated and connected users are disconnected and so forced to use the new configuration
+            * `0` New objects are added, existing ones are updated. This is the default
+            * `1` New objects are added, existing ones are not modified
+            * `2` New objects are added, existing ones are updated and connected users are disconnected and so forced to use the new configuration
    get:
      tags:
        - maintenance
      summary: Load data from path
-      description: 'Restores SFTPGo data from a JSON backup file on the server. Users, folders and admins will be restored one by one and the restore is stopped if a user/folder/admin cannot be added or updated, so it could happen a partial restore'
+      description: 'Restores SFTPGo data from a JSON backup file on the server. Objects will be restored one by one and the restore is stopped if a object cannot be added or updated, so it could happen a partial restore'
      operationId: loaddata_from_file
      parameters:
        - in: query
@ -2760,7 +2760,7 @@ paths:
      tags:
        - maintenance
      summary: Load data
-      description: 'Restores SFTPGo data from a JSON backup. Users, folders and admins will be restored one by one and the restore is stopped if a user/folder/admin cannot be added or updated, so it could happen a partial restore'
+      description: 'Restores SFTPGo data from a JSON backup. Objects will be restored one by one and the restore is stopped if a object cannot be added or updated, so it could happen a partial restore'
      operationId: loaddata_from_request_body
      requestBody:
        required: true