591 lines
16 KiB
Go
591 lines
16 KiB
Go
// SiYuan - Build Your Eternal Digital Garden
|
|
// Copyright (c) 2020-present, b3log.org
|
|
//
|
|
// This program is free software: you can redistribute it and/or modify
|
|
// it under the terms of the GNU Affero General Public License as published by
|
|
// the Free Software Foundation, either version 3 of the License, or
|
|
// (at your option) any later version.
|
|
//
|
|
// This program is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU Affero General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU Affero General Public License
|
|
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
|
|
package model
|
|
|
|
import (
|
|
"bytes"
|
|
"crypto/md5"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"io/fs"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/88250/gulu"
|
|
"github.com/dustin/go-humanize"
|
|
"github.com/siyuan-note/encryption"
|
|
"github.com/siyuan-note/siyuan/kernel/filesys"
|
|
"github.com/siyuan-note/siyuan/kernel/util"
|
|
)
|
|
|
|
type Backup struct {
|
|
Size int64 `json:"size"`
|
|
HSize string `json:"hSize"`
|
|
Updated string `json:"updated"`
|
|
SaveDir string `json:"saveDir"` // 本地备份数据存放目录路径
|
|
}
|
|
|
|
type Sync struct {
|
|
Size int64 `json:"size"`
|
|
HSize string `json:"hSize"`
|
|
Updated string `json:"updated"`
|
|
CloudName string `json:"cloudName"` // 云端同步数据存放目录名
|
|
SaveDir string `json:"saveDir"` // 本地同步数据存放目录路径
|
|
}
|
|
|
|
func RemoveCloudBackup() (err error) {
|
|
err = removeCloudDirPath("backup")
|
|
return
|
|
}
|
|
|
|
func getCloudAvailableBackupSize() (size int64, err error) {
|
|
var sync map[string]interface{}
|
|
var assetSize int64
|
|
sync, _, assetSize, err = getCloudSpaceOSS()
|
|
if nil != err {
|
|
return
|
|
}
|
|
|
|
var syncSize int64
|
|
if nil != sync {
|
|
syncSize = int64(sync["size"].(float64))
|
|
}
|
|
size = int64(Conf.User.UserSiYuanRepoSize) - syncSize - assetSize
|
|
return
|
|
}
|
|
|
|
func GetCloudSpace() (s *Sync, b *Backup, hSize, hAssetSize, hTotalSize string, err error) {
|
|
var sync, backup map[string]interface{}
|
|
var assetSize int64
|
|
sync, backup, assetSize, err = getCloudSpaceOSS()
|
|
if nil != err {
|
|
return nil, nil, "", "", "", errors.New(Conf.Language(30) + " " + err.Error())
|
|
}
|
|
|
|
var totalSize, syncSize, backupSize int64
|
|
var syncUpdated, backupUpdated string
|
|
if nil != sync {
|
|
syncSize = int64(sync["size"].(float64))
|
|
syncUpdated = sync["updated"].(string)
|
|
}
|
|
s = &Sync{
|
|
Size: syncSize,
|
|
HSize: humanize.Bytes(uint64(syncSize)),
|
|
Updated: syncUpdated,
|
|
}
|
|
|
|
if nil != backup {
|
|
backupSize = int64(backup["size"].(float64))
|
|
backupUpdated = backup["updated"].(string)
|
|
}
|
|
b = &Backup{
|
|
Size: backupSize,
|
|
HSize: humanize.Bytes(uint64(backupSize)),
|
|
Updated: backupUpdated,
|
|
}
|
|
totalSize = syncSize + backupSize + assetSize
|
|
hAssetSize = humanize.Bytes(uint64(assetSize))
|
|
hSize = humanize.Bytes(uint64(totalSize))
|
|
hTotalSize = byteCountSI(int64(Conf.User.UserSiYuanRepoSize))
|
|
return
|
|
}
|
|
|
|
func byteCountSI(b int64) string {
|
|
const unit = 1000
|
|
if b < unit {
|
|
return fmt.Sprintf("%d B", b)
|
|
}
|
|
div, exp := int64(unit), 0
|
|
for n := b / unit; n >= unit; n /= unit {
|
|
div *= unit
|
|
exp++
|
|
}
|
|
return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "kMGTPE"[exp])
|
|
}
|
|
|
|
func GetLocalBackup() (ret *Backup, err error) {
|
|
backupDir := Conf.Backup.GetSaveDir()
|
|
if err = os.MkdirAll(backupDir, 0755); nil != err {
|
|
return
|
|
}
|
|
|
|
backup, err := os.Stat(backupDir)
|
|
ret = &Backup{
|
|
Updated: backup.ModTime().Format("2006-01-02 15:04:05"),
|
|
SaveDir: Conf.Backup.GetSaveDir(),
|
|
}
|
|
return
|
|
}
|
|
|
|
func RecoverLocalBackup() (err error) {
|
|
if "" == Conf.E2EEPasswd {
|
|
return errors.New(Conf.Language(11))
|
|
}
|
|
|
|
data := util.AESDecrypt(Conf.E2EEPasswd)
|
|
data, _ = hex.DecodeString(string(data))
|
|
passwd := string(data)
|
|
|
|
CloseWatchAssets()
|
|
defer WatchAssets()
|
|
|
|
// 使用备份恢复时自动暂停同步，避免刚刚恢复后的数据又被同步覆盖 https://github.com/siyuan-note/siyuan/issues/4773
|
|
syncEnabled := Conf.Sync.Enabled
|
|
Conf.Sync.Enabled = false
|
|
Conf.Save()
|
|
|
|
filesys.ReleaseAllFileLocks()
|
|
|
|
util.PushEndlessProgress(Conf.Language(63))
|
|
util.LogInfof("starting recovery...")
|
|
start := time.Now()
|
|
|
|
decryptedDataDir, err := decryptDataDir(passwd)
|
|
if nil != err {
|
|
return
|
|
}
|
|
|
|
newDataDir := filepath.Join(util.WorkspaceDir, "data.new")
|
|
os.RemoveAll(newDataDir)
|
|
if err = os.MkdirAll(newDataDir, 0755); nil != err {
|
|
util.ClearPushProgress(100)
|
|
return
|
|
}
|
|
|
|
if err = stableCopy(decryptedDataDir, newDataDir); nil != err {
|
|
util.ClearPushProgress(100)
|
|
return
|
|
}
|
|
|
|
oldDataDir := filepath.Join(util.WorkspaceDir, "data.old")
|
|
if err = os.RemoveAll(oldDataDir); nil != err {
|
|
util.ClearPushProgress(100)
|
|
return
|
|
}
|
|
|
|
// 备份恢复时生成历史 https://github.com/siyuan-note/siyuan/issues/4752
|
|
if gulu.File.IsExist(util.DataDir) {
|
|
var historyDir string
|
|
historyDir, err = util.GetHistoryDir("backup")
|
|
if nil != err {
|
|
util.LogErrorf("get history dir failed: %s", err)
|
|
util.ClearPushProgress(100)
|
|
return
|
|
}
|
|
|
|
var dirs []os.DirEntry
|
|
dirs, err = os.ReadDir(util.DataDir)
|
|
if nil != err {
|
|
util.LogErrorf("read dir [%s] failed: %s", util.DataDir, err)
|
|
util.ClearPushProgress(100)
|
|
return
|
|
}
|
|
for _, dir := range dirs {
|
|
from := filepath.Join(util.DataDir, dir.Name())
|
|
to := filepath.Join(historyDir, dir.Name())
|
|
if err = os.Rename(from, to); nil != err {
|
|
util.LogErrorf("rename [%s] to [%s] failed: %s", from, to, err)
|
|
util.ClearPushProgress(100)
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
if gulu.File.IsExist(util.DataDir) {
|
|
if err = os.RemoveAll(util.DataDir); nil != err {
|
|
util.LogErrorf("remove [%s] failed: %s", util.DataDir, err)
|
|
util.ClearPushProgress(100)
|
|
return
|
|
}
|
|
}
|
|
|
|
if err = os.Rename(newDataDir, util.DataDir); nil != err {
|
|
util.ClearPushProgress(100)
|
|
util.LogErrorf("rename data dir from [%s] to [%s] failed: %s", newDataDir, util.DataDir, err)
|
|
return
|
|
}
|
|
|
|
elapsed := time.Now().Sub(start).Seconds()
|
|
size, _ := util.SizeOfDirectory(util.DataDir, false)
|
|
sizeStr := humanize.Bytes(uint64(size))
|
|
util.LogInfof("recovered backup [size=%s] in [%.2fs]", sizeStr, elapsed)
|
|
|
|
util.PushEndlessProgress(Conf.Language(62))
|
|
time.Sleep(2 * time.Second)
|
|
RefreshFileTree()
|
|
if syncEnabled {
|
|
func() {
|
|
time.Sleep(5 * time.Second)
|
|
util.PushMsg(Conf.Language(134), 7000)
|
|
}()
|
|
}
|
|
return
|
|
}
|
|
|
|
func CreateLocalBackup() (err error) {
|
|
if "" == Conf.E2EEPasswd {
|
|
return errors.New(Conf.Language(11))
|
|
}
|
|
|
|
defer util.ClearPushProgress(100)
|
|
util.PushEndlessProgress(Conf.Language(22))
|
|
|
|
WaitForWritingFiles()
|
|
|
|
filesys.ReleaseAllFileLocks()
|
|
|
|
util.LogInfof("creating backup...")
|
|
start := time.Now()
|
|
data := util.AESDecrypt(Conf.E2EEPasswd)
|
|
data, _ = hex.DecodeString(string(data))
|
|
passwd := string(data)
|
|
encryptedDataDir, err := encryptDataDir(passwd)
|
|
if nil != err {
|
|
util.LogErrorf("encrypt data dir failed: %s", err)
|
|
err = errors.New(fmt.Sprintf(Conf.Language(23), formatErrorMsg(err)))
|
|
return
|
|
}
|
|
|
|
newBackupDir := Conf.Backup.GetSaveDir() + ".new"
|
|
os.RemoveAll(newBackupDir)
|
|
if err = os.MkdirAll(newBackupDir, 0755); nil != err {
|
|
err = errors.New(fmt.Sprintf(Conf.Language(23), formatErrorMsg(err)))
|
|
return
|
|
}
|
|
|
|
if err = stableCopy(encryptedDataDir, newBackupDir); nil != err {
|
|
util.LogErrorf("copy encrypted data dir from [%s] to [%s] failed: %s", encryptedDataDir, newBackupDir, err)
|
|
err = errors.New(fmt.Sprintf(Conf.Language(23), formatErrorMsg(err)))
|
|
return
|
|
}
|
|
|
|
err = genCloudIndex(newBackupDir, map[string]bool{})
|
|
if nil != err {
|
|
return
|
|
}
|
|
|
|
conf := map[string]interface{}{"updated": time.Now().UnixMilli()}
|
|
data, err = gulu.JSON.MarshalJSON(conf)
|
|
if nil != err {
|
|
util.LogErrorf("marshal backup conf.json failed: %s", err)
|
|
} else {
|
|
confPath := filepath.Join(newBackupDir, "conf.json")
|
|
if err = os.WriteFile(confPath, data, 0644); nil != err {
|
|
util.LogErrorf("write backup conf.json [%s] failed: %s", confPath, err)
|
|
}
|
|
}
|
|
|
|
oldBackupDir := Conf.Backup.GetSaveDir() + ".old"
|
|
os.RemoveAll(oldBackupDir)
|
|
|
|
backupDir := Conf.Backup.GetSaveDir()
|
|
if gulu.File.IsExist(backupDir) {
|
|
if err = os.Rename(backupDir, oldBackupDir); nil != err {
|
|
util.LogErrorf("rename backup dir from [%s] to [%s] failed: %s", backupDir, oldBackupDir, err)
|
|
err = errors.New(fmt.Sprintf(Conf.Language(23), formatErrorMsg(err)))
|
|
return
|
|
}
|
|
}
|
|
if err = os.Rename(newBackupDir, backupDir); nil != err {
|
|
util.LogErrorf("rename backup dir from [%s] to [%s] failed: %s", newBackupDir, backupDir, err)
|
|
err = errors.New(fmt.Sprintf(Conf.Language(23), formatErrorMsg(err)))
|
|
return
|
|
}
|
|
os.RemoveAll(oldBackupDir)
|
|
elapsed := time.Now().Sub(start).Seconds()
|
|
size, _ := util.SizeOfDirectory(backupDir, false)
|
|
sizeStr := humanize.Bytes(uint64(size))
|
|
util.LogInfof("created backup [size=%s] in [%.2fs]", sizeStr, elapsed)
|
|
|
|
util.PushEndlessProgress(Conf.Language(21))
|
|
time.Sleep(2 * time.Second)
|
|
return
|
|
}
|
|
|
|
func DownloadBackup() (err error) {
|
|
// 使用索引文件进行解密验证 https://github.com/siyuan-note/siyuan/issues/3789
|
|
var tmpFetchedFiles int
|
|
var tmpTransferSize uint64
|
|
err = ossDownload0(util.TempDir+"/backup", "backup", "/"+pathJSON, &tmpFetchedFiles, &tmpTransferSize, false)
|
|
if nil != err {
|
|
return
|
|
}
|
|
data, err := os.ReadFile(filepath.Join(util.TempDir, "/backup/"+pathJSON))
|
|
if nil != err {
|
|
return
|
|
}
|
|
passwdData, _ := hex.DecodeString(string(util.AESDecrypt(Conf.E2EEPasswd)))
|
|
passwd := string(passwdData)
|
|
data, err = encryption.AESGCMDecryptBinBytes(data, passwd)
|
|
if nil != err {
|
|
err = errors.New(Conf.Language(28))
|
|
return
|
|
}
|
|
|
|
localDirPath := Conf.Backup.GetSaveDir()
|
|
util.PushEndlessProgress(Conf.Language(68))
|
|
start := time.Now()
|
|
fetchedFilesCount, transferSize, _, err := ossDownload(localDirPath, "backup", false)
|
|
if nil == err {
|
|
elapsed := time.Now().Sub(start).Seconds()
|
|
util.LogInfof("downloaded backup [fetchedFiles=%d, transferSize=%s] in [%.2fs]", fetchedFilesCount, humanize.Bytes(transferSize), elapsed)
|
|
util.PushEndlessProgress(Conf.Language(69))
|
|
}
|
|
return
|
|
}
|
|
|
|
func UploadBackup() (err error) {
|
|
defer util.ClearPushProgress(100)
|
|
|
|
if err = checkUploadBackup(); nil != err {
|
|
return
|
|
}
|
|
|
|
localDirPath := Conf.Backup.GetSaveDir()
|
|
util.PushEndlessProgress(Conf.Language(61))
|
|
util.LogInfof("uploading backup...")
|
|
start := time.Now()
|
|
wroteFiles, transferSize, err := ossUpload(localDirPath, "backup", "", false)
|
|
if nil == err {
|
|
elapsed := time.Now().Sub(start).Seconds()
|
|
util.LogInfof("uploaded backup [wroteFiles=%d, transferSize=%s] in [%.2fs]", wroteFiles, humanize.Bytes(transferSize), elapsed)
|
|
util.PushEndlessProgress(Conf.Language(41))
|
|
time.Sleep(2 * time.Second)
|
|
return
|
|
}
|
|
err = errors.New(formatErrorMsg(err))
|
|
return
|
|
}
|
|
|
|
var pathJSON = fmt.Sprintf("%x", md5.Sum([]byte("paths.json"))) // 6952277a5a37c17aa6a7c6d86cd507b1
|
|
|
|
func encryptDataDir(passwd string) (encryptedDataDir string, err error) {
|
|
encryptedDataDir = filepath.Join(util.WorkspaceDir, "incremental", "backup-encrypt")
|
|
if err = os.RemoveAll(encryptedDataDir); nil != err {
|
|
return
|
|
}
|
|
if err = os.MkdirAll(encryptedDataDir, 0755); nil != err {
|
|
return
|
|
}
|
|
|
|
ctime := map[string]time.Time{}
|
|
metaJSON := map[string]string{}
|
|
filepath.Walk(util.DataDir, func(path string, info fs.FileInfo, _ error) error {
|
|
if util.DataDir == path {
|
|
return nil
|
|
}
|
|
|
|
if isCloudSkipFile(path, info) {
|
|
if info.IsDir() {
|
|
return filepath.SkipDir
|
|
}
|
|
return nil
|
|
}
|
|
|
|
plainP := strings.TrimPrefix(path, util.DataDir+string(os.PathSeparator))
|
|
p := plainP
|
|
parts := strings.Split(p, string(os.PathSeparator))
|
|
buf := bytes.Buffer{}
|
|
for i, part := range parts {
|
|
buf.WriteString(fmt.Sprintf("%x", sha256.Sum256([]byte(part)))[:7])
|
|
if i < len(parts)-1 {
|
|
buf.WriteString(string(os.PathSeparator))
|
|
}
|
|
}
|
|
p = buf.String()
|
|
metaJSON[filepath.ToSlash(p)] = filepath.ToSlash(plainP)
|
|
p = encryptedDataDir + string(os.PathSeparator) + p
|
|
|
|
if info.IsDir() {
|
|
if err = os.MkdirAll(p, 0755); nil != err {
|
|
return io.EOF
|
|
}
|
|
if fi, err0 := os.Stat(path); nil == err0 {
|
|
ctime[p] = fi.ModTime()
|
|
}
|
|
} else {
|
|
if err = os.MkdirAll(filepath.Dir(p), 0755); nil != err {
|
|
return io.EOF
|
|
}
|
|
|
|
f, err0 := os.Create(p)
|
|
if nil != err0 {
|
|
util.LogErrorf("create file [%s] failed: %s", p, err0)
|
|
err = err0
|
|
return io.EOF
|
|
}
|
|
data, err0 := os.ReadFile(path)
|
|
if nil != err0 {
|
|
util.LogErrorf("read file [%s] failed: %s", path, err0)
|
|
err = err0
|
|
return io.EOF
|
|
}
|
|
data, err0 = encryption.AESGCMEncryptBinBytes(data, passwd)
|
|
if nil != err0 {
|
|
util.LogErrorf("encrypt file [%s] failed: %s", path, err0)
|
|
err = errors.New("encrypt file failed")
|
|
return io.EOF
|
|
}
|
|
if _, err0 = f.Write(data); nil != err0 {
|
|
util.LogErrorf("write file [%s] failed: %s", p, err0)
|
|
err = err0
|
|
return io.EOF
|
|
}
|
|
if err0 = f.Close(); nil != err0 {
|
|
util.LogErrorf("close file [%s] failed: %s", p, err0)
|
|
err = err0
|
|
return io.EOF
|
|
}
|
|
|
|
fi, err0 := os.Stat(path)
|
|
if nil != err0 {
|
|
util.LogErrorf("stat file [%s] failed: %s", path, err0)
|
|
err = err0
|
|
return io.EOF
|
|
}
|
|
ctime[p] = fi.ModTime()
|
|
}
|
|
return nil
|
|
})
|
|
if nil != err {
|
|
return
|
|
}
|
|
|
|
for p, t := range ctime {
|
|
if err = os.Chtimes(p, t, t); nil != err {
|
|
return
|
|
}
|
|
}
|
|
|
|
// 检查文件是否全部已经编入索引
|
|
err = filepath.Walk(encryptedDataDir, func(path string, info fs.FileInfo, _ error) error {
|
|
if encryptedDataDir == path {
|
|
return nil
|
|
}
|
|
|
|
path = strings.TrimPrefix(path, encryptedDataDir+string(os.PathSeparator))
|
|
path = filepath.ToSlash(path)
|
|
if _, ok := metaJSON[path]; !ok {
|
|
util.LogErrorf("not found backup path in meta [%s]", path)
|
|
return errors.New(Conf.Language(27))
|
|
}
|
|
return nil
|
|
})
|
|
|
|
if nil != err {
|
|
return
|
|
}
|
|
|
|
data, err := gulu.JSON.MarshalJSON(metaJSON)
|
|
if nil != err {
|
|
return
|
|
}
|
|
data, err = encryption.AESGCMEncryptBinBytes(data, passwd)
|
|
if nil != err {
|
|
return "", errors.New("encrypt file failed")
|
|
}
|
|
meta := filepath.Join(encryptedDataDir, pathJSON)
|
|
if err = gulu.File.WriteFileSafer(meta, data, 0644); nil != err {
|
|
return
|
|
}
|
|
return
|
|
}
|
|
|
|
func decryptDataDir(passwd string) (decryptedDataDir string, err error) {
|
|
decryptedDataDir = filepath.Join(util.WorkspaceDir, "incremental", "backup-decrypt")
|
|
if err = os.RemoveAll(decryptedDataDir); nil != err {
|
|
return
|
|
}
|
|
|
|
backupDir := Conf.Backup.GetSaveDir()
|
|
meta := filepath.Join(backupDir, pathJSON)
|
|
data, err := os.ReadFile(meta)
|
|
if nil != err {
|
|
return
|
|
}
|
|
data, err = encryption.AESGCMDecryptBinBytes(data, passwd)
|
|
if nil != err {
|
|
return "", errors.New(Conf.Language(40))
|
|
}
|
|
|
|
metaJSON := map[string]string{}
|
|
if err = gulu.JSON.UnmarshalJSON(data, &metaJSON); nil != err {
|
|
return
|
|
}
|
|
|
|
modTimes := map[string]time.Time{}
|
|
err = filepath.Walk(backupDir, func(path string, info fs.FileInfo, _ error) error {
|
|
if backupDir == path || pathJSON == info.Name() || strings.HasSuffix(info.Name(), ".json") {
|
|
return nil
|
|
}
|
|
|
|
encryptedP := strings.TrimPrefix(path, backupDir+string(os.PathSeparator))
|
|
encryptedP = filepath.ToSlash(encryptedP)
|
|
plainP := filepath.Join(decryptedDataDir, metaJSON[encryptedP])
|
|
plainP = filepath.FromSlash(plainP)
|
|
|
|
if info.IsDir() {
|
|
if err = os.MkdirAll(plainP, 0755); nil != err {
|
|
return io.EOF
|
|
}
|
|
} else {
|
|
if err = os.MkdirAll(filepath.Dir(plainP), 0755); nil != err {
|
|
return io.EOF
|
|
}
|
|
|
|
var err0 error
|
|
data, err0 = os.ReadFile(path)
|
|
if nil != err0 {
|
|
util.LogErrorf("read file [%s] failed: %s", path, err0)
|
|
err = err0
|
|
return io.EOF
|
|
}
|
|
data, err0 = encryption.AESGCMDecryptBinBytes(data, passwd)
|
|
if nil != err0 {
|
|
util.LogErrorf("decrypt file [%s] failed: %s", path, err0)
|
|
err = errors.New(Conf.Language(40))
|
|
return io.EOF
|
|
}
|
|
if err0 = os.WriteFile(plainP, data, 0644); nil != err0 {
|
|
util.LogErrorf("write file [%s] failed: %s", plainP, err0)
|
|
err = err0
|
|
return io.EOF
|
|
}
|
|
}
|
|
|
|
fi, err0 := os.Stat(path)
|
|
if nil != err0 {
|
|
util.LogErrorf("stat file [%s] failed: %s", path, err0)
|
|
err = err0
|
|
return io.EOF
|
|
}
|
|
modTimes[plainP] = fi.ModTime()
|
|
return nil
|
|
})
|
|
|
|
for plainP, modTime := range modTimes {
|
|
if err = os.Chtimes(plainP, modTime, modTime); nil != err {
|
|
return
|
|
}
|
|
}
|
|
return
|
|
}
|