loaddata: improve shares restore
usage and timestamps are now preserved
This commit is contained in:
Nicola Murino
parent
f2480ce5c9
commit
015aa36c56
10 changed files with 159 additions and 33 deletions
|
|
@ -1160,10 +1160,18 @@ func (p *BoltProvider) addShare(share *Share) error {
|
|||
return err
|
||||
}
|
||||
share.ID = int64(id)
|
||||
share.CreatedAt = util.GetTimeAsMsSinceEpoch(time.Now())
|
||||
share.UpdatedAt = util.GetTimeAsMsSinceEpoch(time.Now())
|
||||
share.LastUseAt = 0
|
||||
share.UsedTokens = 0
|
||||
if !share.IsRestore {
|
||||
share.CreatedAt = util.GetTimeAsMsSinceEpoch(time.Now())
|
||||
share.UpdatedAt = share.CreatedAt
|
||||
share.LastUseAt = 0
|
||||
share.UsedTokens = 0
|
||||
}
|
||||
if share.CreatedAt == 0 {
|
||||
share.CreatedAt = util.GetTimeAsMsSinceEpoch(time.Now())
|
||||
}
|
||||
if share.UpdatedAt == 0 {
|
||||
share.UpdatedAt = share.CreatedAt
|
||||
}
|
||||
if err := p.userExistsInternal(tx, share.Username); err != nil {
|
||||
return util.NewValidationError(fmt.Sprintf("related user %#v does not exists", share.Username))
|
||||
}
|
||||
|
|
@ -1200,10 +1208,18 @@ func (p *BoltProvider) updateShare(share *Share) error {
|
|||
|
||||
share.ID = oldObject.ID
|
||||
share.ShareID = oldObject.ShareID
|
||||
share.UsedTokens = oldObject.UsedTokens
|
||||
share.CreatedAt = oldObject.CreatedAt
|
||||
share.LastUseAt = oldObject.LastUseAt
|
||||
share.UpdatedAt = util.GetTimeAsMsSinceEpoch(time.Now())
|
||||
if !share.IsRestore {
|
||||
share.UsedTokens = oldObject.UsedTokens
|
||||
share.CreatedAt = oldObject.CreatedAt
|
||||
share.LastUseAt = oldObject.LastUseAt
|
||||
share.UpdatedAt = util.GetTimeAsMsSinceEpoch(time.Now())
|
||||
}
|
||||
if share.CreatedAt == 0 {
|
||||
share.CreatedAt = util.GetTimeAsMsSinceEpoch(time.Now())
|
||||
}
|
||||
if share.UpdatedAt == 0 {
|
||||
share.UpdatedAt = share.CreatedAt
|
||||
}
|
||||
if err := p.userExistsInternal(tx, share.Username); err != nil {
|
||||
return util.NewValidationError(fmt.Sprintf("related user %#v does not exists", share.Username))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1103,10 +1103,18 @@ func (p *MemoryProvider) addShare(share *Share) error {
|
|||
if _, err := p.userExistsInternal(share.Username); err != nil {
|
||||
return util.NewValidationError(fmt.Sprintf("related user %#v does not exists", share.Username))
|
||||
}
|
||||
share.CreatedAt = util.GetTimeAsMsSinceEpoch(time.Now())
|
||||
share.UpdatedAt = util.GetTimeAsMsSinceEpoch(time.Now())
|
||||
share.LastUseAt = 0
|
||||
share.UsedTokens = 0
|
||||
if !share.IsRestore {
|
||||
share.CreatedAt = util.GetTimeAsMsSinceEpoch(time.Now())
|
||||
share.UpdatedAt = share.CreatedAt
|
||||
share.LastUseAt = 0
|
||||
share.UsedTokens = 0
|
||||
}
|
||||
if share.CreatedAt == 0 {
|
||||
share.CreatedAt = util.GetTimeAsMsSinceEpoch(time.Now())
|
||||
}
|
||||
if share.UpdatedAt == 0 {
|
||||
share.UpdatedAt = share.CreatedAt
|
||||
}
|
||||
p.dbHandle.shares[share.ShareID] = share.getACopy()
|
||||
p.dbHandle.sharesIDs = append(p.dbHandle.sharesIDs, share.ShareID)
|
||||
sort.Strings(p.dbHandle.sharesIDs)
|
||||
|
|
@ -1133,10 +1141,18 @@ func (p *MemoryProvider) updateShare(share *Share) error {
|
|||
}
|
||||
share.ID = s.ID
|
||||
share.ShareID = s.ShareID
|
||||
share.UsedTokens = s.UsedTokens
|
||||
share.CreatedAt = s.CreatedAt
|
||||
share.LastUseAt = s.LastUseAt
|
||||
share.UpdatedAt = util.GetTimeAsMsSinceEpoch(time.Now())
|
||||
if !share.IsRestore {
|
||||
share.UsedTokens = s.UsedTokens
|
||||
share.CreatedAt = s.CreatedAt
|
||||
share.LastUseAt = s.LastUseAt
|
||||
share.UpdatedAt = util.GetTimeAsMsSinceEpoch(time.Now())
|
||||
}
|
||||
if share.CreatedAt == 0 {
|
||||
share.CreatedAt = util.GetTimeAsMsSinceEpoch(time.Now())
|
||||
}
|
||||
if share.UpdatedAt == 0 {
|
||||
share.UpdatedAt = share.CreatedAt
|
||||
}
|
||||
p.dbHandle.shares[share.ShareID] = share.getACopy()
|
||||
return nil
|
||||
}
|
||||
|
|
@ -1346,6 +1362,7 @@ func (p *MemoryProvider) restoreShares(dump *BackupData) error {
|
|||
for _, share := range dump.Shares {
|
||||
s, err := p.shareExists(share.ShareID, "")
|
||||
share := share // pin
|
||||
share.IsRestore = true
|
||||
if err == nil {
|
||||
share.ID = s.ID
|
||||
err = UpdateShare(&share, ActionExecutorSystem, "")
|
||||
|
|
|
|||
|
|
@ -54,6 +54,10 @@ type Share struct {
|
|||
UsedTokens int `json:"used_tokens,omitempty"`
|
||||
// Limit the share availability to these IPs/CIDR networks
|
||||
AllowFrom []string `json:"allow_from,omitempty"`
|
||||
// set for restores, we don't have to validate the expiration date
|
||||
// otherwise we fail to restore existing shares and we have to insert
|
||||
// all the previous values with no modifications
|
||||
IsRestore bool `json:"-"`
|
||||
}
|
||||
|
||||
// GetScopeAsString returns the share's scope as string.
|
||||
|
|
@ -210,7 +214,7 @@ func (s *Share) validate() error {
|
|||
return err
|
||||
}
|
||||
if s.ExpiresAt > 0 {
|
||||
if s.ExpiresAt < util.GetTimeAsMsSinceEpoch(time.Now()) {
|
||||
if !s.IsRestore && s.ExpiresAt < util.GetTimeAsMsSinceEpoch(time.Now()) {
|
||||
return util.NewValidationError("expiration must be in the future")
|
||||
}
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -93,9 +93,23 @@ func sqlCommonAddShare(share *Share, dbHandle *sql.DB) error {
|
|||
}
|
||||
defer stmt.Close()
|
||||
|
||||
usedTokens := 0
|
||||
createdAt := util.GetTimeAsMsSinceEpoch(time.Now())
|
||||
updatedAt := createdAt
|
||||
lastUseAt := int64(0)
|
||||
if share.IsRestore {
|
||||
usedTokens = share.UsedTokens
|
||||
if share.CreatedAt > 0 {
|
||||
createdAt = share.CreatedAt
|
||||
}
|
||||
if share.UpdatedAt > 0 {
|
||||
updatedAt = share.UpdatedAt
|
||||
}
|
||||
lastUseAt = share.LastUseAt
|
||||
}
|
||||
_, err = stmt.ExecContext(ctx, share.ShareID, share.Name, share.Description, share.Scope,
|
||||
string(paths), util.GetTimeAsMsSinceEpoch(time.Now()), util.GetTimeAsMsSinceEpoch(time.Now()),
|
||||
share.LastUseAt, share.ExpiresAt, share.Password, share.MaxTokens, allowFrom, user.ID)
|
||||
string(paths), createdAt, updatedAt, lastUseAt, share.ExpiresAt, share.Password,
|
||||
share.MaxTokens, usedTokens, allowFrom, user.ID)
|
||||
return err
|
||||
}
|
||||
|
||||
|
|
@ -125,7 +139,12 @@ func sqlCommonUpdateShare(share *Share, dbHandle *sql.DB) error {
|
|||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout)
|
||||
defer cancel()
|
||||
q := getUpdateShareQuery()
|
||||
var q string
|
||||
if share.IsRestore {
|
||||
q = getUpdateShareRestoreQuery()
|
||||
} else {
|
||||
q = getUpdateShareQuery()
|
||||
}
|
||||
stmt, err := dbHandle.PrepareContext(ctx, q)
|
||||
if err != nil {
|
||||
providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
|
||||
|
|
@ -133,9 +152,21 @@ func sqlCommonUpdateShare(share *Share, dbHandle *sql.DB) error {
|
|||
}
|
||||
defer stmt.Close()
|
||||
|
||||
_, err = stmt.ExecContext(ctx, share.Name, share.Description, share.Scope, string(paths),
|
||||
util.GetTimeAsMsSinceEpoch(time.Now()), share.ExpiresAt, share.Password, share.MaxTokens,
|
||||
allowFrom, user.ID, share.ShareID)
|
||||
if share.IsRestore {
|
||||
if share.CreatedAt == 0 {
|
||||
share.CreatedAt = util.GetTimeAsMsSinceEpoch(time.Now())
|
||||
}
|
||||
if share.UpdatedAt == 0 {
|
||||
share.UpdatedAt = share.CreatedAt
|
||||
}
|
||||
_, err = stmt.ExecContext(ctx, share.Name, share.Description, share.Scope, string(paths),
|
||||
share.CreatedAt, share.UpdatedAt, share.LastUseAt, share.ExpiresAt, share.Password, share.MaxTokens,
|
||||
share.UsedTokens, allowFrom, user.ID, share.ShareID)
|
||||
} else {
|
||||
_, err = stmt.ExecContext(ctx, share.Name, share.Description, share.Scope, string(paths),
|
||||
util.GetTimeAsMsSinceEpoch(time.Now()), share.ExpiresAt, share.Password, share.MaxTokens,
|
||||
allowFrom, user.ID, share.ShareID)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -82,11 +82,19 @@ func getDumpSharesQuery() string {
|
|||
|
||||
func getAddShareQuery() string {
|
||||
return fmt.Sprintf(`INSERT INTO %v (share_id,name,description,scope,paths,created_at,updated_at,last_use_at,
|
||||
expires_at,password,max_tokens,used_tokens,allow_from,user_id) VALUES (%v,%v,%v,%v,%v,%v,%v,%v,%v,%v,%v,0,%v,%v)`,
|
||||
expires_at,password,max_tokens,used_tokens,allow_from,user_id) VALUES (%v,%v,%v,%v,%v,%v,%v,%v,%v,%v,%v,%v,%v,%v)`,
|
||||
sqlTableShares, sqlPlaceholders[0], sqlPlaceholders[1],
|
||||
sqlPlaceholders[2], sqlPlaceholders[3], sqlPlaceholders[4], sqlPlaceholders[5], sqlPlaceholders[6],
|
||||
sqlPlaceholders[7], sqlPlaceholders[8], sqlPlaceholders[9], sqlPlaceholders[10], sqlPlaceholders[11],
|
||||
sqlPlaceholders[12])
|
||||
sqlPlaceholders[12], sqlPlaceholders[13])
|
||||
}
|
||||
|
||||
func getUpdateShareRestoreQuery() string {
|
||||
return fmt.Sprintf(`UPDATE %v SET name=%v,description=%v,scope=%v,paths=%v,created_at=%v,updated_at=%v,
|
||||
last_use_at=%v,expires_at=%v,password=%v,max_tokens=%v,used_tokens=%v,allow_from=%v,user_id=%v WHERE share_id = %v`, sqlTableShares,
|
||||
sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3], sqlPlaceholders[4],
|
||||
sqlPlaceholders[5], sqlPlaceholders[6], sqlPlaceholders[7], sqlPlaceholders[8], sqlPlaceholders[9],
|
||||
sqlPlaceholders[10], sqlPlaceholders[11], sqlPlaceholders[12], sqlPlaceholders[13])
|
||||
}
|
||||
|
||||
func getUpdateShareQuery() string {
|
||||
|
|
|
|||
2
go.mod
2
go.mod
|
|
@ -7,7 +7,7 @@ require (
|
|||
github.com/Azure/azure-storage-blob-go v0.14.0
|
||||
github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962
|
||||
github.com/alexedwards/argon2id v0.0.0-20210511081203-7d35d68092b8
|
||||
github.com/aws/aws-sdk-go v1.42.12
|
||||
github.com/aws/aws-sdk-go v1.42.13
|
||||
github.com/cockroachdb/cockroach-go/v2 v2.2.4
|
||||
github.com/eikenb/pipeat v0.0.0-20210603033007-44fc3ffce52b
|
||||
github.com/fclairamb/ftpserverlib v0.16.0
|
||||
|
|
|
|||
4
go.sum
4
go.sum
|
|
@ -137,8 +137,8 @@ github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZo
|
|||
github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
|
||||
github.com/aws/aws-sdk-go v1.38.68/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
|
||||
github.com/aws/aws-sdk-go v1.40.34/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
|
||||
github.com/aws/aws-sdk-go v1.42.12 h1:zVrAgi3/HuMPygZknc+f2KAHcn+Zuq767857hnHBMPA=
|
||||
github.com/aws/aws-sdk-go v1.42.12/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
|
||||
github.com/aws/aws-sdk-go v1.42.13 h1:+Nx87T+Bjiq2XybxK6vI98cTEBPLE/hILuZyEenlyEg=
|
||||
github.com/aws/aws-sdk-go v1.42.13/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
|
||||
github.com/aws/aws-sdk-go-v2 v1.7.0/go.mod h1:tb9wi5s61kTDA5qCkcDbt3KRVV74GGslQkl/DRdX/P4=
|
||||
github.com/aws/aws-sdk-go-v2 v1.9.0/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.7.0/go.mod h1:w9+nMZ7soXCe5nT46Ri354SNhXDQ6v+V5wqDjnZE+GY=
|
||||
|
|
|
|||
|
|
@ -254,6 +254,7 @@ func RestoreShares(shares []dataprovider.Share, inputFile string, mode int, exec
|
|||
) error {
|
||||
for _, share := range shares {
|
||||
share := share // pin
|
||||
share.IsRestore = true
|
||||
s, err := dataprovider.ShareExists(share.ShareID, "")
|
||||
if err == nil {
|
||||
if mode == 1 {
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ import (
|
|||
"github.com/go-chi/render"
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
_ "github.com/lib/pq"
|
||||
"github.com/lithammer/shortuuid/v3"
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
"github.com/mhale/smtpd"
|
||||
"github.com/pquerna/otp"
|
||||
|
|
@ -4270,6 +4271,54 @@ func TestDefenderAPIErrors(t *testing.T) {
|
|||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestRestoreShares(t *testing.T) {
|
||||
// shares should be restored preserving the UsedTokens, CreatedAt, LastUseAt, UpdatedAt,
|
||||
// and ExpiresAt, so an expired share can be restored while we cannot create an already
|
||||
// expired share
|
||||
user, _, err := httpdtest.AddUser(getTestUser(), http.StatusCreated)
|
||||
assert.NoError(t, err)
|
||||
share := dataprovider.Share{
|
||||
ShareID: shortuuid.New(),
|
||||
Name: "share name",
|
||||
Description: "share description",
|
||||
Scope: dataprovider.ShareScopeRead,
|
||||
Paths: []string{"/"},
|
||||
Username: user.Username,
|
||||
CreatedAt: util.GetTimeAsMsSinceEpoch(time.Now().Add(-144 * time.Hour)),
|
||||
UpdatedAt: util.GetTimeAsMsSinceEpoch(time.Now().Add(-96 * time.Hour)),
|
||||
LastUseAt: util.GetTimeAsMsSinceEpoch(time.Now().Add(-64 * time.Hour)),
|
||||
ExpiresAt: util.GetTimeAsMsSinceEpoch(time.Now().Add(-48 * time.Hour)),
|
||||
MaxTokens: 10,
|
||||
UsedTokens: 8,
|
||||
AllowFrom: []string{"127.0.0.0/8"},
|
||||
}
|
||||
backupData := dataprovider.BackupData{}
|
||||
backupData.Shares = append(backupData.Shares, share)
|
||||
backupContent, err := json.Marshal(backupData)
|
||||
assert.NoError(t, err)
|
||||
_, _, err = httpdtest.LoaddataFromPostBody(backupContent, "0", "0", http.StatusOK)
|
||||
assert.NoError(t, err)
|
||||
shareGet, err := dataprovider.ShareExists(share.ShareID, user.Username)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, share, shareGet)
|
||||
|
||||
share.CreatedAt = util.GetTimeAsMsSinceEpoch(time.Now().Add(-142 * time.Hour))
|
||||
share.UpdatedAt = util.GetTimeAsMsSinceEpoch(time.Now().Add(-92 * time.Hour))
|
||||
share.LastUseAt = util.GetTimeAsMsSinceEpoch(time.Now().Add(-62 * time.Hour))
|
||||
share.UsedTokens = 6
|
||||
backupData.Shares = []dataprovider.Share{share}
|
||||
backupContent, err = json.Marshal(backupData)
|
||||
assert.NoError(t, err)
|
||||
_, _, err = httpdtest.LoaddataFromPostBody(backupContent, "0", "0", http.StatusOK)
|
||||
assert.NoError(t, err)
|
||||
shareGet, err = dataprovider.ShareExists(share.ShareID, user.Username)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, share, shareGet)
|
||||
|
||||
_, err = httpdtest.RemoveUser(user, http.StatusOK)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestLoaddataFromPostBody(t *testing.T) {
|
||||
mappedPath := filepath.Join(os.TempDir(), "restored_folder")
|
||||
folderName := filepath.Base(mappedPath)
|
||||
|
|
|
|||
|
|
@ -2721,14 +2721,14 @@ paths:
|
|||
- 2
|
||||
description: |
|
||||
Mode:
|
||||
* `0` New users/admins/API keys are added, existing ones are updated. This is the default
|
||||
* `1` New users/admins/API keys are added, existing ones are not modified
|
||||
* `2` New users/admins/API keys are added, existing ones are updated and connected users are disconnected and so forced to use the new configuration
|
||||
* `0` New objects are added, existing ones are updated. This is the default
|
||||
* `1` New objects are added, existing ones are not modified
|
||||
* `2` New objects are added, existing ones are updated and connected users are disconnected and so forced to use the new configuration
|
||||
get:
|
||||
tags:
|
||||
- maintenance
|
||||
summary: Load data from path
|
||||
description: 'Restores SFTPGo data from a JSON backup file on the server. Users, folders and admins will be restored one by one and the restore is stopped if a user/folder/admin cannot be added or updated, so it could happen a partial restore'
|
||||
description: 'Restores SFTPGo data from a JSON backup file on the server. Objects will be restored one by one and the restore is stopped if a object cannot be added or updated, so it could happen a partial restore'
|
||||
operationId: loaddata_from_file
|
||||
parameters:
|
||||
- in: query
|
||||
|
|
@ -2760,7 +2760,7 @@ paths:
|
|||
tags:
|
||||
- maintenance
|
||||
summary: Load data
|
||||
description: 'Restores SFTPGo data from a JSON backup. Users, folders and admins will be restored one by one and the restore is stopped if a user/folder/admin cannot be added or updated, so it could happen a partial restore'
|
||||
description: 'Restores SFTPGo data from a JSON backup. Objects will be restored one by one and the restore is stopped if a object cannot be added or updated, so it could happen a partial restore'
|
||||
operationId: loaddata_from_request_body
|
||||
requestBody:
|
||||
required: true
|
||||
|
|
|
|||
Loading…
Reference in a new issue