Add support for persisting remote files metadata

internal/api/collection.go

@@ -20,6 +20,25 @@ func (c *Client) GetCollections(ctx context.Context, sinceTime int64) ([]Collect
 			Message:    r.String(),
 		}
 	}
-
 	return res.Collections, err
 }
+
+func (c *Client) GetFiles(ctx context.Context, collectionID, sinceTime int64) ([]File, bool, error) {
+	var res struct {
+		Files   []File `json:"diff"`
+		HasMore bool   `json:"hasMore"`
+	}
+	r, err := c.restClient.R().
+		SetContext(ctx).
+		SetQueryParam("sinceTime", strconv.FormatInt(sinceTime, 10)).
+		SetQueryParam("collectionID", strconv.FormatInt(collectionID, 10)).
+		SetResult(&res).
+		Get("/collections/v2/diff")
+	if r.IsError() {
+		return nil, false, &ApiError{
+			StatusCode: r.StatusCode(),
+			Message:    r.String(),
+		}
+	}
+	return res.Files, res.HasMore, err
+}

internal/api/files.go

@@ -0,0 +1 @@
+package api

pkg/collections.go

@@ -5,7 +5,9 @@ import (
 	"cli-go/pkg/model"
 	"cli-go/utils/encoding"
 	"context"
+	"encoding/json"
 	"fmt"
+	"log"
 	"strconv"
 )
 
@@ -45,3 +47,73 @@ func (c *ClICtrl) fetchRemoteCollections(ctx context.Context, info model.Account
 	}
 	return nil
 }
+
+func (c *ClICtrl) fetchRemoteFiles(ctx context.Context, info model.Account) error {
+	albums, err := c.getRemoteAlbums(ctx)
+	if err != nil {
+		return err
+	}
+	for _, album := range albums {
+		if album.IsDeleted {
+			log.Printf("Skipping album %s as it is deleted", album.AlbumName)
+			continue
+		}
+		lastSyncTime, lastSyncTimeErr := c.GetInt64ConfigValue(ctx, fmt.Sprintf(model.CollectionsFileSyncKeyFmt, album.ID))
+		if lastSyncTimeErr != nil {
+			return lastSyncTimeErr
+		}
+		isFirstSync := lastSyncTime == 0
+		for {
+			if lastSyncTime == album.LastUpdatedAt {
+				break
+			}
+			files, hasMore, err := c.Client.GetFiles(ctx, album.ID, lastSyncTime)
+			if err != nil {
+				return err
+			}
+			maxUpdated := lastSyncTime
+			for _, file := range files {
+				if file.UpdationTime > maxUpdated {
+					maxUpdated = file.UpdationTime
+				}
+				if isFirstSync && file.IsDeleted {
+					// on first sync, no need to sync delete markers
+					continue
+				}
+				fileJson := encoding.MustMarshalJSON(file)
+				putErr := c.PutValue(ctx, model.RemoteFiles, []byte(strconv.FormatInt(file.ID, 10)), fileJson)
+				if putErr != nil {
+					return putErr
+				}
+			}
+			if !hasMore {
+				maxUpdated = album.LastUpdatedAt
+			}
+			if maxUpdated > lastSyncTime || !hasMore {
+				err = c.PutConfigValue(ctx, fmt.Sprintf(model.CollectionsFileSyncKeyFmt, album.ID), []byte(strconv.FormatInt(maxUpdated, 10)))
+				if err != nil {
+					return fmt.Errorf("failed to update last sync time: %s", err)
+				} else {
+					lastSyncTime = maxUpdated
+				}
+			}
+		}
+	}
+	return nil
+}
+func (c *ClICtrl) getRemoteAlbums(ctx context.Context) ([]model.Album, error) {
+	albums := make([]model.Album, 0)
+	albumBytes, err := c.GetAllValues(ctx, model.RemoteAlbums)
+	if err != nil {
+		return nil, err
+	}
+	for _, albumJson := range albumBytes {
+		album := model.Album{}
+		err = json.Unmarshal(albumJson, &album)
+		if err != nil {
+			return nil, err
+		}
+		albums = append(albums, album)
+	}
+	return albums, nil
+}

pkg/model/constants.go

@@ -3,9 +3,9 @@ package model
 type PhotosStore string
 
 const (
-	KVConfig     PhotosStore = "kvConfig"
-	RemoteAlbums PhotosStore = "remoteAlbums"
-	RemoteFiles  PhotosStore = "remoteFiles"
+	KVConfig     PhotosStore = "akvConfig"
+	RemoteAlbums PhotosStore = "aremoteAlbums"
+	RemoteFiles  PhotosStore = "aremoteFiles"
 )
 
 const (

pkg/remote_sync.go

@@ -24,6 +24,10 @@ func (c *ClICtrl) SyncAccount(account model.Account) error {
 	if err != nil {
 		log.Printf("Error fetching collections: %s", err)
 	}
+	err = c.fetchRemoteFiles(ctx, account)
+	if err != nil {
+		log.Printf("Error fetching files: %s", err)
+	}
 	return nil
 }
 

pkg/store.go

@@ -19,6 +19,21 @@ func GetDB(path string) (*bolt.DB, error) {
 	return db, err
 }
 
+func (c *ClICtrl) GetInt64ConfigValue(ctx context.Context, key string) (int64, error) {
+	value, err := c.GetConfigValue(ctx, key)
+	if err != nil {
+		return 0, err
+	}
+	var result int64
+	if value != nil {
+		result, err = strconv.ParseInt(string(value), 10, 64)
+		if err != nil {
+			return 0, err
+		}
+	}
+	return result, nil
+}
+
 func (c *ClICtrl) GetConfigValue(ctx context.Context, key string) ([]byte, error) {
 	var value []byte
 	err := c.DB.View(func(tx *bolt.Tx) error {
@@ -32,19 +47,20 @@ func (c *ClICtrl) GetConfigValue(ctx context.Context, key string) ([]byte, error
 	return value, err
 }
 
-func (c *ClICtrl) GetInt64ConfigValue(ctx context.Context, key string) (int64, error) {
-	value, err := c.GetConfigValue(ctx, key)
-	if err != nil {
-		return 0, err
-	}
-	var result int64
-	if value != nil {
-		result, err = strconv.ParseInt(string(value), 10, 64)
+func (c *ClICtrl) GetAllValues(ctx context.Context, store model.PhotosStore) ([][]byte, error) {
+	result := make([][]byte, 0)
+	err := c.DB.View(func(tx *bolt.Tx) error {
+		kvBucket, err := getAccountStore(ctx, tx, store)
 		if err != nil {
-			return 0, err
+			return err
 		}
-	}
-	return result, nil
+		kvBucket.ForEach(func(k, v []byte) error {
+			result = append(result, v)
+			return nil
+		})
+		return nil
+	})
+	return result, err
 }
 
 func (c *ClICtrl) PutConfigValue(ctx context.Context, key string, value []byte) error {
@@ -65,7 +81,6 @@ func (c *ClICtrl) PutValue(ctx context.Context, store model.PhotosStore, key []b
 		return kvBucket.Put(key, value)
 	})
 }
-
 func getAccountStore(ctx context.Context, tx *bolt.Tx, storeType model.PhotosStore) (*bolt.Bucket, error) {
 	accountId := ctx.Value("account_id").(string)
 	accountBucket := tx.Bucket([]byte(accountId))