0ct0pu5/ente
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Refactor

Browse source
This commit is contained in:
Neeraj Gupta 2023-09-27 11:02:36 +05:30
parent 9b50fbfa35
commit 86385a1a2f
9 changed files with 210 additions and 219 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
pkg/account.go
View file

@ -45,7 +45,7 @@ func (c *ClICtrl) AddAccount(cxt context.Context) {
if verifyEmail || srpAttr.IsEmailMFAEnabled {
authResponse, flowErr = c.validateEmail(cxt, email)
} else {
authResponse, keyEncKey, flowErr = c.signInViaPassword(cxt, email, srpAttr)
authResponse, keyEncKey, flowErr = c.signInViaPassword(cxt, srpAttr)
}
if flowErr != nil {
return

0
pkg/controller.go → pkg/cli.go
View file

128
pkg/collections.go
View file

@ -1,128 +0,0 @@
package pkg
import (
"cli-go/pkg/model"
"cli-go/utils/encoding"
"context"
"encoding/json"
"fmt"
"log"
"strconv"
"time"
)
func (c *ClICtrl) fetchRemoteCollections(ctx context.Context) error {
lastSyncTime, err2 := c.GetInt64ConfigValue(ctx, model.CollectionsSyncKey)
if err2 != nil {
return err2
}
collections, err := c.Client.GetCollections(ctx, lastSyncTime)
if err != nil {
return fmt.Errorf("failed to get collections: %s", err)
}
maxUpdated := lastSyncTime
for _, collection := range collections {
if lastSyncTime == 0 && collection.IsDeleted {
continue
}
album, mapErr := c.mapCollectionToAlbum(ctx, collection)
if mapErr != nil {
return mapErr
}
if album.LastUpdatedAt > maxUpdated {
maxUpdated = album.LastUpdatedAt
}
albumJson := encoding.MustMarshalJSON(album)
putErr := c.PutValue(ctx, model.RemoteAlbums, []byte(strconv.FormatInt(album.ID, 10)), albumJson)
if putErr != nil {
return putErr
}
}
if maxUpdated > lastSyncTime {
err = c.PutConfigValue(ctx, model.CollectionsSyncKey, []byte(strconv.FormatInt(maxUpdated, 10)))
if err != nil {
return fmt.Errorf("failed to update last sync time: %s", err)
}
}
return nil
}
func (c *ClICtrl) fetchRemoteFiles(ctx context.Context) error {
albums, err := c.getRemoteAlbums(ctx)
if err != nil {
return err
}
for _, album := range albums {
if album.IsDeleted {
log.Printf("Skipping album %s as it is deleted", album.AlbumName)
continue
}
lastSyncTime, lastSyncTimeErr := c.GetInt64ConfigValue(ctx, fmt.Sprintf(model.CollectionsFileSyncKeyFmt, album.ID))
if lastSyncTimeErr != nil {
return lastSyncTimeErr
}
isFirstSync := lastSyncTime == 0
for {
if lastSyncTime == album.LastUpdatedAt {
break
}
if !isFirstSync {
t := time.UnixMicro(lastSyncTime)
log.Printf("Fetching files for album %s from %v\n", album.AlbumName, t)
}
files, hasMore, err := c.Client.GetFiles(ctx, album.ID, lastSyncTime)
if err != nil {
return err
}
maxUpdated := lastSyncTime
for _, file := range files {
if file.UpdationTime > maxUpdated {
maxUpdated = file.UpdationTime
}
if isFirstSync && file.IsDeleted {
// on first sync, no need to sync delete markers
continue
}
photoFile, err := c.mapApiFileToPhotoFile(ctx, album, file)
if err != nil {
return err
}
fileJson := encoding.MustMarshalJSON(photoFile)
putErr := c.PutValue(ctx, model.RemoteFiles, []byte(strconv.FormatInt(file.ID, 10)), fileJson)
if putErr != nil {
return putErr
}
}
if !hasMore {
maxUpdated = album.LastUpdatedAt
}
if maxUpdated > lastSyncTime || !hasMore {
err = c.PutConfigValue(ctx, fmt.Sprintf(model.CollectionsFileSyncKeyFmt, album.ID), []byte(strconv.FormatInt(maxUpdated, 10)))
if err != nil {
return fmt.Errorf("failed to update last sync time: %s", err)
} else {
lastSyncTime = maxUpdated
}
}
}
}
return nil
}
func (c *ClICtrl) getRemoteAlbums(ctx context.Context) ([]model.RemoteAlbum, error) {
albums := make([]model.RemoteAlbum, 0)
albumBytes, err := c.GetAllValues(ctx, model.RemoteAlbums)
if err != nil {
return nil, err
}
for _, albumJson := range albumBytes {
album := model.RemoteAlbum{}
err = json.Unmarshal(albumJson, &album)
if err != nil {
return nil, err
}
albums = append(albums, album)
}
return albums, nil
}

19
pkg/download.go
View file

@ -1,9 +1,7 @@
package pkg
import (
"cli-go/pkg/model"
"context"
"encoding/json"
"fmt"
"log"
"os"
@ -28,20 +26,3 @@ func (c *ClICtrl) initiateDownload(ctx context.Context) error {
}
return nil
}
func (c *ClICtrl) getRemoteFiles(ctx context.Context) ([]model.RemoteFile, error) {
files := make([]model.RemoteFile, 0)
fileBytes, err := c.GetAllValues(ctx, model.RemoteFiles)
if err != nil {
return nil, err
}
for _, fileJson := range fileBytes {
file := model.RemoteFile{}
err = json.Unmarshal(fileJson, &file)
if err != nil {
return nil, err
}
files = append(files, file)
}
return files, nil
}

31
pkg/export.go
View file

@ -1,31 +0,0 @@
package pkg
import (
"context"
"fmt"
"log"
)
func (c *ClICtrl) StartSync() error {
accounts, err := c.GetAccounts(context.Background())
if err != nil {
return err
}
if len(accounts) == 0 {
fmt.Printf("No accounts to sync\n")
return nil
}
for _, account := range accounts {
log.SetPrefix(fmt.Sprintf("[%s-%s] ", account.App, account.Email))
log.Println("start sync")
err = c.SyncAccount(account)
if err != nil {
fmt.Printf("Error syncing account %s: %s\n", account.Email, err)
return err
} else {
log.Println("sync done")
}
}
return nil
}

159
pkg/remote_sync.go
View file

@ -2,59 +2,144 @@ package pkg
import (
"cli-go/pkg/model"
"cli-go/utils/encoding"
"context"
"encoding/base64"
"encoding/json"
"fmt"
bolt "go.etcd.io/bbolt"
"log"
"strconv"
"time"
)
func (c *ClICtrl) SyncAccount(account model.Account) error {
secretInfo, err := c.KeyHolder.LoadSecrets(account, c.CliKey)
if err != nil {
return err
func (c *ClICtrl) fetchRemoteCollections(ctx context.Context) error {
lastSyncTime, err2 := c.GetInt64ConfigValue(ctx, model.CollectionsSyncKey)
if err2 != nil {
return err2
}
ctx := c.buildRequestContext(context.Background(), account)
err = createDataBuckets(c.DB, account)
collections, err := c.Client.GetCollections(ctx, lastSyncTime)
if err != nil {
return err
return fmt.Errorf("failed to get collections: %s", err)
}
c.Client.AddToken(account.AccountKey(), base64.URLEncoding.EncodeToString(secretInfo.Token))
err = c.fetchRemoteCollections(ctx)
if err != nil {
log.Printf("Error fetching collections: %s", err)
maxUpdated := lastSyncTime
for _, collection := range collections {
if lastSyncTime == 0 && collection.IsDeleted {
continue
}
album, mapErr := c.mapCollectionToAlbum(ctx, collection)
if mapErr != nil {
return mapErr
}
if album.LastUpdatedAt > maxUpdated {
maxUpdated = album.LastUpdatedAt
}
albumJson := encoding.MustMarshalJSON(album)
putErr := c.PutValue(ctx, model.RemoteAlbums, []byte(strconv.FormatInt(album.ID, 10)), albumJson)
if putErr != nil {
return putErr
}
}
err = c.fetchRemoteFiles(ctx)
if err != nil {
log.Printf("Error fetching files: %s", err)
}
downloadErr := c.initiateDownload(ctx)
if downloadErr != nil {
log.Printf("Error downloading files: %s", downloadErr)
return downloadErr
if maxUpdated > lastSyncTime {
err = c.PutConfigValue(ctx, model.CollectionsSyncKey, []byte(strconv.FormatInt(maxUpdated, 10)))
if err != nil {
return fmt.Errorf("failed to update last sync time: %s", err)
}
}
return nil
}
func (c *ClICtrl) buildRequestContext(ctx context.Context, account model.Account) context.Context {
ctx = context.WithValue(ctx, "app", string(account.App))
ctx = context.WithValue(ctx, "account_id", account.AccountKey())
ctx = context.WithValue(ctx, "user_id", account.UserID)
return ctx
}
func createDataBuckets(db *bolt.DB, account model.Account) error {
return db.Update(func(tx *bolt.Tx) error {
dataBucket, err := tx.CreateBucketIfNotExists([]byte(account.AccountKey()))
if err != nil {
return fmt.Errorf("create bucket: %s", err)
func (c *ClICtrl) fetchRemoteFiles(ctx context.Context) error {
albums, err := c.getRemoteAlbums(ctx)
if err != nil {
return err
}
for _, album := range albums {
if album.IsDeleted {
log.Printf("Skipping album %s as it is deleted", album.AlbumName)
continue
}
for _, subBucket := range []model.PhotosStore{model.KVConfig, model.RemoteAlbums, model.RemoteFiles} {
_, err := dataBucket.CreateBucketIfNotExists([]byte(subBucket))
lastSyncTime, lastSyncTimeErr := c.GetInt64ConfigValue(ctx, fmt.Sprintf(model.CollectionsFileSyncKeyFmt, album.ID))
if lastSyncTimeErr != nil {
return lastSyncTimeErr
}
isFirstSync := lastSyncTime == 0
for {
if lastSyncTime == album.LastUpdatedAt {
break
}
if !isFirstSync {
t := time.UnixMicro(lastSyncTime)
log.Printf("Fetching files for album %s from %v\n", album.AlbumName, t)
}
files, hasMore, err := c.Client.GetFiles(ctx, album.ID, lastSyncTime)
if err != nil {
return err
}
maxUpdated := lastSyncTime
for _, file := range files {
if file.UpdationTime > maxUpdated {
maxUpdated = file.UpdationTime
}
if isFirstSync && file.IsDeleted {
// on first sync, no need to sync delete markers
continue
}
photoFile, err := c.mapApiFileToPhotoFile(ctx, album, file)
if err != nil {
return err
}
fileJson := encoding.MustMarshalJSON(photoFile)
putErr := c.PutValue(ctx, model.RemoteFiles, []byte(strconv.FormatInt(file.ID, 10)), fileJson)
if putErr != nil {
return putErr
}
}
if !hasMore {
maxUpdated = album.LastUpdatedAt
}
if maxUpdated > lastSyncTime || !hasMore {
err = c.PutConfigValue(ctx, fmt.Sprintf(model.CollectionsFileSyncKeyFmt, album.ID), []byte(strconv.FormatInt(maxUpdated, 10)))
if err != nil {
return fmt.Errorf("failed to update last sync time: %s", err)
} else {
lastSyncTime = maxUpdated
}
}
}
return nil
})
}
return nil
}
func (c *ClICtrl) getRemoteAlbums(ctx context.Context) ([]model.RemoteAlbum, error) {
albums := make([]model.RemoteAlbum, 0)
albumBytes, err := c.GetAllValues(ctx, model.RemoteAlbums)
if err != nil {
return nil, err
}
for _, albumJson := range albumBytes {
album := model.RemoteAlbum{}
err = json.Unmarshal(albumJson, &album)
if err != nil {
return nil, err
}
albums = append(albums, album)
}
return albums, nil
}
func (c *ClICtrl) getRemoteFiles(ctx context.Context) ([]model.RemoteFile, error) {
files := make([]model.RemoteFile, 0)
fileBytes, err := c.GetAllValues(ctx, model.RemoteFiles)
if err != nil {
return nil, err
}
for _, fileJson := range fileBytes {
file := model.RemoteFile{}
err = json.Unmarshal(fileJson, &file)
if err != nil {
return nil, err
}
files = append(files, file)
}
return files, nil
}

2
pkg/sign_in.go
View file

@ -13,7 +13,7 @@ import (
"github.com/kong/go-srp"
)
func (c *ClICtrl) signInViaPassword(ctx context.Context, email string, srpAttr *api.SRPAttributes) (*api.AuthorizationResponse, []byte, error) {
func (c *ClICtrl) signInViaPassword(ctx context.Context, srpAttr *api.SRPAttributes) (*api.AuthorizationResponse, []byte, error) {
for {
// CLI prompt for password
password, flowErr := internal.GetSensitiveField("Enter password")

4
pkg/store.go
View file

@ -20,7 +20,7 @@ func GetDB(path string) (*bolt.DB, error) {
}
func (c *ClICtrl) GetInt64ConfigValue(ctx context.Context, key string) (int64, error) {
value, err := c.GetConfigValue(ctx, key)
value, err := c.getConfigValue(ctx, key)
if err != nil {
return 0, err
}
@ -34,7 +34,7 @@ func (c *ClICtrl) GetInt64ConfigValue(ctx context.Context, key string) (int64, e
return result, nil
}
func (c *ClICtrl) GetConfigValue(ctx context.Context, key string) ([]byte, error) {
func (c *ClICtrl) getConfigValue(ctx context.Context, key string) ([]byte, error) {
var value []byte
err := c.DB.View(func(tx *bolt.Tx) error {
kvBucket, err := getAccountStore(ctx, tx, model.KVConfig)

84
pkg/sync.go Normal file
View file

@ -0,0 +1,84 @@
package pkg
import (
"cli-go/pkg/model"
"context"
"encoding/base64"
"fmt"
bolt "go.etcd.io/bbolt"
"log"
)
func (c *ClICtrl) StartSync() error {
accounts, err := c.GetAccounts(context.Background())
if err != nil {
return err
}
if len(accounts) == 0 {
fmt.Printf("No accounts to sync\n")
return nil
}
for _, account := range accounts {
log.SetPrefix(fmt.Sprintf("[%s-%s] ", account.App, account.Email))
log.Println("start sync")
err = c.SyncAccount(account)
if err != nil {
fmt.Printf("Error syncing account %s: %s\n", account.Email, err)
return err
} else {
log.Println("sync done")
}
}
return nil
}
func (c *ClICtrl) SyncAccount(account model.Account) error {
secretInfo, err := c.KeyHolder.LoadSecrets(account, c.CliKey)
if err != nil {
return err
}
ctx := c.buildRequestContext(context.Background(), account)
err = createDataBuckets(c.DB, account)
if err != nil {
return err
}
c.Client.AddToken(account.AccountKey(), base64.URLEncoding.EncodeToString(secretInfo.Token))
err = c.fetchRemoteCollections(ctx)
if err != nil {
log.Printf("Error fetching collections: %s", err)
}
err = c.fetchRemoteFiles(ctx)
if err != nil {
log.Printf("Error fetching files: %s", err)
}
downloadErr := c.initiateDownload(ctx)
if downloadErr != nil {
log.Printf("Error downloading files: %s", downloadErr)
return downloadErr
}
return nil
}
func (c *ClICtrl) buildRequestContext(ctx context.Context, account model.Account) context.Context {
ctx = context.WithValue(ctx, "app", string(account.App))
ctx = context.WithValue(ctx, "account_id", account.AccountKey())
ctx = context.WithValue(ctx, "user_id", account.UserID)
return ctx
}
func createDataBuckets(db *bolt.DB, account model.Account) error {
return db.Update(func(tx *bolt.Tx) error {
dataBucket, err := tx.CreateBucketIfNotExists([]byte(account.AccountKey()))
if err != nil {
return fmt.Errorf("create bucket: %s", err)
}
for _, subBucket := range []model.PhotosStore{model.KVConfig, model.RemoteAlbums, model.RemoteFiles} {
_, err := dataBucket.CreateBucketIfNotExists([]byte(subBucket))
if err != nil {
return err
}
}
return nil
})
}