Add support for persisting remote files metadata
This commit is contained in:
parent
cab929da2b
commit
8aafeccd1d
6 changed files with 129 additions and 18 deletions
|
@ -20,6 +20,25 @@ func (c *Client) GetCollections(ctx context.Context, sinceTime int64) ([]Collect
|
|||
Message: r.String(),
|
||||
}
|
||||
}
|
||||
|
||||
return res.Collections, err
|
||||
}
|
||||
|
||||
func (c *Client) GetFiles(ctx context.Context, collectionID, sinceTime int64) ([]File, bool, error) {
|
||||
var res struct {
|
||||
Files []File `json:"diff"`
|
||||
HasMore bool `json:"hasMore"`
|
||||
}
|
||||
r, err := c.restClient.R().
|
||||
SetContext(ctx).
|
||||
SetQueryParam("sinceTime", strconv.FormatInt(sinceTime, 10)).
|
||||
SetQueryParam("collectionID", strconv.FormatInt(collectionID, 10)).
|
||||
SetResult(&res).
|
||||
Get("/collections/v2/diff")
|
||||
if r.IsError() {
|
||||
return nil, false, &ApiError{
|
||||
StatusCode: r.StatusCode(),
|
||||
Message: r.String(),
|
||||
}
|
||||
}
|
||||
return res.Files, res.HasMore, err
|
||||
}
|
||||
|
|
1
internal/api/files.go
Normal file
1
internal/api/files.go
Normal file
|
@ -0,0 +1 @@
|
|||
package api
|
|
@ -5,7 +5,9 @@ import (
|
|||
"cli-go/pkg/model"
|
||||
"cli-go/utils/encoding"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
|
@ -45,3 +47,73 @@ func (c *ClICtrl) fetchRemoteCollections(ctx context.Context, info model.Account
|
|||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *ClICtrl) fetchRemoteFiles(ctx context.Context, info model.Account) error {
|
||||
albums, err := c.getRemoteAlbums(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, album := range albums {
|
||||
if album.IsDeleted {
|
||||
log.Printf("Skipping album %s as it is deleted", album.AlbumName)
|
||||
continue
|
||||
}
|
||||
lastSyncTime, lastSyncTimeErr := c.GetInt64ConfigValue(ctx, fmt.Sprintf(model.CollectionsFileSyncKeyFmt, album.ID))
|
||||
if lastSyncTimeErr != nil {
|
||||
return lastSyncTimeErr
|
||||
}
|
||||
isFirstSync := lastSyncTime == 0
|
||||
for {
|
||||
if lastSyncTime == album.LastUpdatedAt {
|
||||
break
|
||||
}
|
||||
files, hasMore, err := c.Client.GetFiles(ctx, album.ID, lastSyncTime)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
maxUpdated := lastSyncTime
|
||||
for _, file := range files {
|
||||
if file.UpdationTime > maxUpdated {
|
||||
maxUpdated = file.UpdationTime
|
||||
}
|
||||
if isFirstSync && file.IsDeleted {
|
||||
// on first sync, no need to sync delete markers
|
||||
continue
|
||||
}
|
||||
fileJson := encoding.MustMarshalJSON(file)
|
||||
putErr := c.PutValue(ctx, model.RemoteFiles, []byte(strconv.FormatInt(file.ID, 10)), fileJson)
|
||||
if putErr != nil {
|
||||
return putErr
|
||||
}
|
||||
}
|
||||
if !hasMore {
|
||||
maxUpdated = album.LastUpdatedAt
|
||||
}
|
||||
if maxUpdated > lastSyncTime || !hasMore {
|
||||
err = c.PutConfigValue(ctx, fmt.Sprintf(model.CollectionsFileSyncKeyFmt, album.ID), []byte(strconv.FormatInt(maxUpdated, 10)))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to update last sync time: %s", err)
|
||||
} else {
|
||||
lastSyncTime = maxUpdated
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (c *ClICtrl) getRemoteAlbums(ctx context.Context) ([]model.Album, error) {
|
||||
albums := make([]model.Album, 0)
|
||||
albumBytes, err := c.GetAllValues(ctx, model.RemoteAlbums)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, albumJson := range albumBytes {
|
||||
album := model.Album{}
|
||||
err = json.Unmarshal(albumJson, &album)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
albums = append(albums, album)
|
||||
}
|
||||
return albums, nil
|
||||
}
|
||||
|
|
|
@ -3,9 +3,9 @@ package model
|
|||
type PhotosStore string
|
||||
|
||||
const (
|
||||
KVConfig PhotosStore = "kvConfig"
|
||||
RemoteAlbums PhotosStore = "remoteAlbums"
|
||||
RemoteFiles PhotosStore = "remoteFiles"
|
||||
KVConfig PhotosStore = "akvConfig"
|
||||
RemoteAlbums PhotosStore = "aremoteAlbums"
|
||||
RemoteFiles PhotosStore = "aremoteFiles"
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
|
@ -24,6 +24,10 @@ func (c *ClICtrl) SyncAccount(account model.Account) error {
|
|||
if err != nil {
|
||||
log.Printf("Error fetching collections: %s", err)
|
||||
}
|
||||
err = c.fetchRemoteFiles(ctx, account)
|
||||
if err != nil {
|
||||
log.Printf("Error fetching files: %s", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
43
pkg/store.go
43
pkg/store.go
|
@ -19,19 +19,6 @@ func GetDB(path string) (*bolt.DB, error) {
|
|||
return db, err
|
||||
}
|
||||
|
||||
func (c *ClICtrl) GetConfigValue(ctx context.Context, key string) ([]byte, error) {
|
||||
var value []byte
|
||||
err := c.DB.View(func(tx *bolt.Tx) error {
|
||||
kvBucket, err := getAccountStore(ctx, tx, model.KVConfig)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
value = kvBucket.Get([]byte(key))
|
||||
return nil
|
||||
})
|
||||
return value, err
|
||||
}
|
||||
|
||||
func (c *ClICtrl) GetInt64ConfigValue(ctx context.Context, key string) (int64, error) {
|
||||
value, err := c.GetConfigValue(ctx, key)
|
||||
if err != nil {
|
||||
|
@ -47,6 +34,35 @@ func (c *ClICtrl) GetInt64ConfigValue(ctx context.Context, key string) (int64, e
|
|||
return result, nil
|
||||
}
|
||||
|
||||
func (c *ClICtrl) GetConfigValue(ctx context.Context, key string) ([]byte, error) {
|
||||
var value []byte
|
||||
err := c.DB.View(func(tx *bolt.Tx) error {
|
||||
kvBucket, err := getAccountStore(ctx, tx, model.KVConfig)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
value = kvBucket.Get([]byte(key))
|
||||
return nil
|
||||
})
|
||||
return value, err
|
||||
}
|
||||
|
||||
func (c *ClICtrl) GetAllValues(ctx context.Context, store model.PhotosStore) ([][]byte, error) {
|
||||
result := make([][]byte, 0)
|
||||
err := c.DB.View(func(tx *bolt.Tx) error {
|
||||
kvBucket, err := getAccountStore(ctx, tx, store)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
kvBucket.ForEach(func(k, v []byte) error {
|
||||
result = append(result, v)
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
})
|
||||
return result, err
|
||||
}
|
||||
|
||||
func (c *ClICtrl) PutConfigValue(ctx context.Context, key string, value []byte) error {
|
||||
return c.DB.Update(func(tx *bolt.Tx) error {
|
||||
kvBucket, err := getAccountStore(ctx, tx, model.KVConfig)
|
||||
|
@ -65,7 +81,6 @@ func (c *ClICtrl) PutValue(ctx context.Context, store model.PhotosStore, key []b
|
|||
return kvBucket.Put(key, value)
|
||||
})
|
||||
}
|
||||
|
||||
func getAccountStore(ctx context.Context, tx *bolt.Tx, storeType model.PhotosStore) (*bolt.Bucket, error) {
|
||||
accountId := ctx.Value("account_id").(string)
|
||||
accountBucket := tx.Bucket([]byte(accountId))
|
||||
|
|
Loading…
Add table
Reference in a new issue