0ct0pu5/siyuan
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

🎨 同步下载支持断点续传 https://github.com/siyuan-note/siyuan/issues/5056

Browse source
This commit is contained in:
Liang Ding 2022-05-31 10:25:06 +08:00
parent 2d96137033
commit cd5c993fbd
No known key found for this signature in database
GPG key ID: 136F30F901A2231D
3 changed files with 50 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
kernel/model/backup.go
View file

@ -344,10 +344,10 @@ func DownloadBackup() (err error) {
localDirPath := Conf.Backup.GetSaveDir()
util.PushEndlessProgress(Conf.Language(68))
start := time.Now()
fetchedFiles, transferSize, err := ossDownload(localDirPath, "backup", false)
fetchedFilesCount, transferSize, _, err := ossDownload(localDirPath, "backup", false)
if nil == err {
elapsed := time.Now().Sub(start).Seconds()
util.LogInfof("downloaded backup [fetchedFiles=%d, transferSize=%s] in [%.2fs]", fetchedFiles, humanize.Bytes(transferSize), elapsed)
util.LogInfof("downloaded backup [fetchedFiles=%d, transferSize=%s] in [%.2fs]", fetchedFilesCount, humanize.Bytes(transferSize), elapsed)
util.PushEndlessProgress(Conf.Language(69))
}
return

19
kernel/model/osssync.go
View file

@ -161,7 +161,7 @@ func listCloudSyncDirOSS() (dirs []map[string]interface{}, size int64, err error
return
}
func ossDownload(localDirPath, cloudDirPath string, bootOrExit bool) (fetchedFiles int, transferSize uint64, err error) {
func ossDownload(localDirPath, cloudDirPath string, bootOrExit bool) (fetchedFilesCount int, transferSize uint64, downloadedFiles []string, err error) {
if !gulu.File.IsExist(localDirPath) {
return
}
@ -187,24 +187,29 @@ func ossDownload(localDirPath, cloudDirPath string, bootOrExit bool) (fetchedFil
waitGroup := &sync.WaitGroup{}
var downloadErr error
poolSize := 4
if poolSize > len(cloudFetches) {
if poolSize > len(cloudFetches)-1 /* 不计入 /.siyuan/conf.json，配置文件最后单独下载 */ {
poolSize = len(cloudFetches)
}
p, _ := ants.NewPoolWithFunc(poolSize, func(arg interface{}) {
defer waitGroup.Done()
if nil != downloadErr {
return // 快速失败
}
fetch := arg.(string)
err = ossDownload0(localDirPath, cloudDirPath, fetch, &fetchedFiles, &transferSize, bootOrExit)
err = ossDownload0(localDirPath, cloudDirPath, fetch, &fetchedFilesCount, &transferSize, bootOrExit)
if nil != err {
downloadErr = err // 仅记录最后一次错误
return
}
downloadedFiles = append(downloadedFiles, fetch)
if needPushProgress {
msg := fmt.Sprintf(Conf.Language(103), fetchedFiles, len(cloudFetches)-fetchedFiles)
util.PushProgress(util.PushProgressCodeProgressed, fetchedFiles, len(cloudFetches), msg)
msg := fmt.Sprintf(Conf.Language(103), fetchedFilesCount, len(cloudFetches)-fetchedFilesCount)
util.PushProgress(util.PushProgressCodeProgressed, fetchedFilesCount, len(cloudFetches), msg)
}
if bootOrExit {
msg := fmt.Sprintf("Downloading data from the cloud %d/%d", fetchedFiles, len(cloudFetches))
msg := fmt.Sprintf("Downloading data from the cloud %d/%d", fetchedFilesCount, len(cloudFetches))
util.IncBootProgress(0, msg)
}
})
@ -224,7 +229,7 @@ func ossDownload(localDirPath, cloudDirPath string, bootOrExit bool) (fetchedFil
return
}
err = ossDownload0(localDirPath, cloudDirPath, "/.siyuan/conf.json", &fetchedFiles, &transferSize, bootOrExit)
err = ossDownload0(localDirPath, cloudDirPath, "/.siyuan/conf.json", &fetchedFilesCount, &transferSize, bootOrExit)
if nil != err {
return
}

39
kernel/model/sync.go
View file

@ -321,12 +321,18 @@ func SyncData(boot, exit, byHand bool) {
return
}
fetchedFiles, transferSize, err := ossDownload(localSyncDirPath, "sync/"+Conf.Sync.CloudName, boot || exit)
fetchedFilesCount, transferSize, downloadedFiles, err := ossDownload(localSyncDirPath, "sync/"+Conf.Sync.CloudName, boot || exit)
if nil != err {
util.PushClearMsg()
msg := fmt.Sprintf(Conf.Language(80), formatErrorMsg(err))
Conf.Sync.Stat = msg
util.PushErrMsg(msg, 7000)
err = syncDirUpsertWorkspaceData(downloadedFiles)
if nil != err {
util.LogErrorf("upsert partially downloaded files to workspace data failed: %s", err)
}
if boot {
BootSyncSucc = 1
}
@ -360,8 +366,8 @@ func SyncData(boot, exit, byHand bool) {
clearEmptyDirs(util.DataDir)
elapsed := time.Now().Sub(start).Seconds()
stat := fmt.Sprintf(Conf.Language(129), fetchedFiles, humanize.Bytes(transferSize)) + fmt.Sprintf(Conf.Language(131), elapsed)
util.LogInfof("sync [cloud=%d, local=%d, fetchedFiles=%d, transferSize=%s] downloaded in [%.2fs]", cloudSyncVer, syncConf.SyncVer, fetchedFiles, humanize.Bytes(transferSize), elapsed)
stat := fmt.Sprintf(Conf.Language(129), fetchedFilesCount, humanize.Bytes(transferSize)) + fmt.Sprintf(Conf.Language(131), elapsed)
util.LogInfof("sync [cloud=%d, local=%d, fetchedFiles=%d, transferSize=%s] downloaded in [%.2fs]", cloudSyncVer, syncConf.SyncVer, fetchedFilesCount, humanize.Bytes(transferSize), elapsed)
Conf.Sync.Downloaded = now
Conf.Sync.Stat = stat
@ -490,6 +496,33 @@ func SetSyncEnable(b bool) (err error) {
var syncLock = sync.Mutex{}
func syncDirUpsertWorkspaceData(downloadedFiles []string) (err error) {
start := time.Now()
modified := map[string]bool{}
syncDir := Conf.Sync.GetSaveDir()
for _, file := range downloadedFiles {
file = filepath.Join(syncDir, file)
modified[file] = true
}
decryptedDataDir, _, err := recoverSyncData(modified)
if nil != err {
util.LogErrorf("decrypt data dir failed: %s", err)
return
}
dataDir := util.DataDir
if err = stableCopy(decryptedDataDir, dataDir); nil != err {
util.LogErrorf("copy decrypted data dir from [%s] to data dir [%s] failed: %s", decryptedDataDir, dataDir, err)
return
}
if elapsed := time.Since(start).Milliseconds(); 5000 < elapsed {
util.LogInfof("sync data to workspace data elapsed [%dms]", elapsed)
}
return
}
// syncDir2WorkspaceData 将 sync 的数据更新到 data 中。
// 1. 删除 data 中冗余的文件
// 2. 将 sync 中新增/修改的文件解密后拷贝到 data 中