Merge remote-tracking branch 'origin/dev' into dev

app/appearance/langs/en_US.json

@@ -866,6 +866,10 @@
     "132": "Uploaded in %.2fs",
     "133": "No changes to local data",
     "134": "In order to prevent the newly restored data from being overwritten by synchronization, the data synchronization function has been automatically suspended",
-    "135": "Please make sure that all devices have been updated to the latest version, and then trigger synchronization after randomly changing a document on the main device, and finally trigger synchronization on other devices"
+    "135": "Please make sure that all devices have been updated to the latest version, and then trigger synchronization after randomly changing a document on the main device, and finally trigger synchronization on other devices",
+    "136": "Initializing data repository key...",
+    "137": "Failed to initialize data repository key",
+    "138": "Data repository key is set",
+    "139": "Data snapshot is being generated, [%s] data files have been processed, [%s] remaining..."
   }
 }

app/appearance/langs/fr_FR.json

@@ -866,6 +866,10 @@
     "132": "Le téléchargement a pris %.2fs",
     "133": "Aucune modification des données locales",
     "134": "Afin d'éviter que les données nouvellement restaurées ne soient écrasées par la synchronisation, la fonction de synchronisation des données a été automatiquement suspendue",
-    "135": "Assurez-vous que tous les appareils ont été mis à jour vers la dernière version, puis déclenchez la synchronisation après avoir modifié de manière aléatoire un document sur l'appareil principal, et enfin déclenchez la synchronisation sur d'autres appareils."
+    "135": "Assurez-vous que tous les appareils ont été mis à jour vers la dernière version, puis déclenchez la synchronisation après avoir modifié de manière aléatoire un document sur l'appareil principal, et enfin déclenchez la synchronisation sur d'autres appareils.",
+    "136": "Initialisation de la clé du référentiel de données...",
+    "137": "Échec de l'initialisation de la clé du référentiel de données",
+    "138": "La clé du référentiel de données est définie",
+    "139": "L'instantané des données est en cours de génération, [%s] fichiers de données ont été traités, [%s] restants..."
   }
 }

app/appearance/langs/zh_CHT.json

@@ -865,6 +865,10 @@
     "132": "上傳耗時 %.2fs",
     "133": "本地數據暫無變更",
     "134": "為避免剛恢復的數據被同步覆蓋，數據同步功能已被自動暫停",
-    "135": "請確保所有設備已經更新到最新版，然後在主力設備上隨意更改一個文檔後觸發同步，最後再到其他設備觸發同步"
+    "135": "請確保所有設備已經更新到最新版，然後在主力設備上隨意更改一個文檔後觸發同步，最後再到其他設備觸發同步",
+    "136": "初始化數據倉庫密鑰...",
+    "137": "初始化數據倉庫密鑰失敗",
+    "138": "數據倉庫密鑰設置完畢",
+    "139": "正在生成數據快照，已處理 [%s] 個數據文件，剩餘待處理 [%s]..."
   }
 }

app/appearance/langs/zh_CN.json

@@ -867,6 +867,10 @@
     "132": "上传耗时 %.2fs",
     "133": "本地数据暂无变更",
     "134": "为避免刚恢复的数据被同步覆盖，数据同步功能已被自动暂停",
-    "135": "请确保所有设备已经更新到最新版，然后在主力设备上随意更改一个文档后触发同步，最后再到其他设备触发同步"
+    "135": "请确保所有设备已经更新到最新版，然后在主力设备上随意更改一个文档后触发同步，最后再到其他设备触发同步",
+    "136": "初始化数据仓库密钥...",
+    "137": "初始化数据仓库密钥失败",
+    "138": "数据仓库密钥设置完毕",
+    "139": "正在生成数据快照，已处理 [%s] 个数据文件，剩余待处理 [%s]..."
   }
 }

kernel/api/router.go

@@ -45,6 +45,7 @@ func ServeAPI(ginServer *gin.Engine) {
 	ginServer.Handle("POST", "/api/system/setAppearanceMode", model.CheckAuth, setAppearanceMode)
 	ginServer.Handle("POST", "/api/system/getSysFonts", model.CheckAuth, getSysFonts)
 	ginServer.Handle("POST", "/api/system/setE2EEPasswd", model.CheckAuth, setE2EEPasswd)
+	ginServer.Handle("POST", "/api/system/initRepoKey", model.CheckAuth, initRepoKey)
 	ginServer.Handle("POST", "/api/system/exit", model.CheckAuth, exit)
 	ginServer.Handle("POST", "/api/system/setUILayout", model.CheckAuth, setUILayout)
 	ginServer.Handle("POST", "/api/system/getConf", model.CheckAuth, getConf)

kernel/api/system.go

@@ -364,7 +364,31 @@ func setE2EEPasswd(c *gin.Context) {
 	util.PushMsg(model.Conf.Language(92), 3000)
 	time.Sleep(1 * time.Second)
 	model.SyncData(false, false, true)
+}
 
+func initRepoKey(c *gin.Context) {
+	ret := gulu.Ret.NewResult()
+	defer c.JSON(http.StatusOK, ret)
+
+	util.PushMsg(model.Conf.Language(136), 1000*7)
+	if err := os.RemoveAll(model.Conf.Repo.GetSaveDir()); nil != err {
+		ret.Code = -1
+		ret.Msg = err.Error()
+		return
+	}
+	if err := os.MkdirAll(model.Conf.Repo.GetSaveDir(), 0755); nil != err {
+		ret.Code = -1
+		ret.Msg = err.Error()
+		return
+	}
+	if err := model.InitRepoKey(); nil != err {
+		ret.Code = -1
+		ret.Msg = model.Conf.Language(137)
+		return
+	}
+
+	time.Sleep(1 * time.Second)
+	util.PushMsg(model.Conf.Language(138), 3000)
 }
 
 func addUIProcess(c *gin.Context) {

kernel/conf/Repo.go

@@ -0,0 +1,35 @@
+// SiYuan - Build Your Eternal Digital Garden
+// Copyright (c) 2020-present, b3log.org
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package conf
+
+import (
+	"path/filepath"
+
+	"github.com/siyuan-note/siyuan/kernel/util"
+)
+
+type Repo struct {
+	Key []byte `json:"key"` // AES 密钥
+}
+
+func NewRepo() *Repo {
+	return &Repo{}
+}
+
+func (*Repo) GetSaveDir() string {
+	return filepath.Join(util.WorkspaceDir, "repo")
+}

kernel/go.mod

@@ -40,7 +40,7 @@ require (
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/qiniu/go-sdk/v7 v7.13.0
 	github.com/radovskyb/watcher v1.0.7
-	github.com/siyuan-note/encryption v0.0.0-20220609141541-7ba697278320
+	github.com/siyuan-note/encryption v0.0.0-20220612074546-f1dd94fe8676
 	github.com/vmihailenco/msgpack/v5 v5.3.5
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673
 	golang.org/x/image v0.0.0-20220601225756-64ec528b34cd
@@ -78,6 +78,7 @@ require (
 	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/juju/errors v0.0.0-20220331221717-b38fca44723b // indirect
+	github.com/klauspost/compress v1.15.6 // indirect
 	github.com/leodido/go-urn v1.2.1 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
@@ -86,7 +87,9 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.2 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/restic/chunker v0.4.0 // indirect
 	github.com/shopspring/decimal v1.3.1 // indirect
+	github.com/siyuan-note/dejavu v0.0.0-20220612144647-aab04039e33b // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/ugorji/go/codec v1.2.7 // indirect
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect

kernel/go.sum

@@ -320,6 +320,8 @@ github.com/juju/testing v0.0.0-20191001232224-ce9dec17d28b/go.mod h1:63prj8cnj0t
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.15.6 h1:6D9PcO8QWu0JyaQ2zUMmu16T1T+zjjEpP91guRsvDfY=
+github.com/klauspost/compress v1.15.6/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
 github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -401,6 +403,8 @@ github.com/qiniu/go-sdk/v7 v7.13.0/go.mod h1:btsaOc8CA3hdVloULfFdDgDc+g4f3TDZEFs
 github.com/qiniu/x v1.10.5/go.mod h1:03Ni9tj+N2h2aKnAz+6N0Xfl8FwMEDRC2PAlxekASDs=
 github.com/radovskyb/watcher v1.0.7 h1:AYePLih6dpmS32vlHfhCeli8127LzkIgwJGcwwe8tUE=
 github.com/radovskyb/watcher v1.0.7/go.mod h1:78okwvY5wPdzcb1UYnip1pvrZNIVEIh/Cm+ZuvsUYIg=
+github.com/restic/chunker v0.4.0 h1:YUPYCUn70MYP7VO4yllypp2SjmsRhRJaad3xKu1QFRw=
+github.com/restic/chunker v0.4.0/go.mod h1:z0cH2BejpW636LXw0R/BGyv+Ey8+m9QGiOanDHItzyw=
 github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
@@ -421,10 +425,14 @@ github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJ
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw=
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/siyuan-note/dejavu v0.0.0-20220612144647-aab04039e33b h1:HQLzy92vLOMy6MR2t1/oEW7LRfKMD2TljMYwwue2i5Q=
+github.com/siyuan-note/dejavu v0.0.0-20220612144647-aab04039e33b/go.mod h1:pZFVF2imlfZMT5w6899ZwXCLKVccZ9GDJzdLHJ+ONik=
 github.com/siyuan-note/encryption v0.0.0-20210811062758-4d08f2d31e37 h1:WvJU9uRS7kaaqnNShIMMtR2Yf8duGmXYJXYGg69EXBs=
 github.com/siyuan-note/encryption v0.0.0-20210811062758-4d08f2d31e37/go.mod h1:hWBdT3FZEzWvIbZpXYJvkSBH2+Z4GvYcOpKpXcZC+zg=
 github.com/siyuan-note/encryption v0.0.0-20220609141541-7ba697278320 h1:CQKC+OpTjzFJjZK9/NY3OYg/gSYmMQM9RsBk5Mznimg=
 github.com/siyuan-note/encryption v0.0.0-20220609141541-7ba697278320/go.mod h1:H8fyqqAbp9XreANjeSbc72zEdFfKTXYN34tc1TjZwtw=
+github.com/siyuan-note/encryption v0.0.0-20220612074546-f1dd94fe8676 h1:QB9TjJQFhXhZ6dAtPpY02DlzHAQm1C+WqZq6OadG8mI=
+github.com/siyuan-note/encryption v0.0.0-20220612074546-f1dd94fe8676/go.mod h1:H8fyqqAbp9XreANjeSbc72zEdFfKTXYN34tc1TjZwtw=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=

kernel/model/conf.go

@@ -70,6 +70,7 @@ type AppConf struct {
 	Search         *conf.Search     `json:"search"`         // 搜索配置
 	Stat           *conf.Stat       `json:"stat"`           // 统计
 	Api            *conf.API        `json:"api"`            // API
+	Repo           *conf.Repo       `json:"repo"`           // 数据仓库
 	Newbie         bool             `json:"newbie"`         // 是否是安装后第一次启动
 }
 
@@ -222,6 +223,10 @@ func InitConf() {
 		Conf.Api = conf.NewAPI()
 	}
 
+	if nil == Conf.Repo {
+		Conf.Repo = conf.NewRepo()
+	}
+
 	if 1440 < Conf.Editor.GenerateHistoryInterval {
 		Conf.Editor.GenerateHistoryInterval = 1440
 	}

kernel/model/repository.go

@@ -0,0 +1,66 @@
+// SiYuan - Build Your Eternal Digital Garden
+// Copyright (c) 2020-present, b3log.org
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package model
+
+import (
+	"crypto/rand"
+	"errors"
+
+	"github.com/siyuan-note/dejavu"
+	"github.com/siyuan-note/encryption"
+	"github.com/siyuan-note/siyuan/kernel/util"
+)
+
+func InitRepoKey() (err error) {
+	randomBytes := make([]byte, 16)
+	_, err = rand.Read(randomBytes)
+	if nil != err {
+		return
+	}
+	password := string(randomBytes)
+	randomBytes = make([]byte, 16)
+	_, err = rand.Read(randomBytes)
+	if nil != err {
+		util.LogErrorf("init repo key failed: %s", err)
+		return
+	}
+	salt := string(randomBytes)
+
+	key, err := encryption.KDF(password, salt)
+	if nil != err {
+		util.LogErrorf("init repo key failed: %s", err)
+		return
+	}
+	Conf.Repo.Key = key
+	Conf.Save()
+	return
+}
+
+func IndexRepo(message string) (err error) {
+	if 1 > len(Conf.Repo.Key) {
+		err = errors.New("repo key is nil")
+		return
+	}
+
+	repo, err := dejavu.NewRepo(util.DataDir, util.RepoDir, Conf.Repo.Key)
+	if nil != err {
+		return
+	}
+
+	_, err = repo.Index(message)
+	return
+}

kernel/util/working.go

@@ -162,6 +162,7 @@ var (
 	WorkspaceDir   string        // 工作空间目录路径
 	ConfDir        string        // 配置目录路径
 	DataDir        string        // 数据目录路径
+	RepoDir        string        // 仓库目录路径
 	TempDir        string        // 临时目录路径
 	LogPath        string        // 配置目录下的日志文件 siyuan.log 路径
 	DBName         = "siyuan.db" // SQLite 数据库文件名
@@ -249,6 +250,7 @@ func initWorkspaceDir(workspaceArg string) {
 
 	ConfDir = filepath.Join(WorkspaceDir, "conf")
 	DataDir = filepath.Join(WorkspaceDir, "data")
+	RepoDir = filepath.Join(WorkspaceDir, "repo")
 	TempDir = filepath.Join(WorkspaceDir, "temp")
 	osTmpDir := filepath.Join(TempDir, "os")
 	os.RemoveAll(osTmpDir)

kernel/util/working_mobile.go

@@ -35,6 +35,7 @@ func BootMobile(container, appDir, workspaceDir, nativeLibDir, privateDataDir, l
 	WorkspaceDir = workspaceDir
 	ConfDir = filepath.Join(workspaceDir, "conf")
 	DataDir = filepath.Join(workspaceDir, "data")
+	RepoDir = filepath.Join(WorkspaceDir, "repo")
 	TempDir = filepath.Join(workspaceDir, "temp")
 	osTmpDir := filepath.Join(TempDir, "os")
 	os.RemoveAll(osTmpDir)