index.go 7.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274
  1. // SiYuan - Build Your Eternal Digital Garden
  2. // Copyright (c) 2020-present, b3log.org
  3. //
  4. // This program is free software: you can redistribute it and/or modify
  5. // it under the terms of the GNU Affero General Public License as published by
  6. // the Free Software Foundation, either version 3 of the License, or
  7. // (at your option) any later version.
  8. //
  9. // This program is distributed in the hope that it will be useful,
  10. // but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12. // GNU Affero General Public License for more details.
  13. //
  14. // You should have received a copy of the GNU Affero General Public License
  15. // along with this program. If not, see <https://www.gnu.org/licenses/>.
  16. package model
  17. import (
  18. "bytes"
  19. "crypto/sha256"
  20. "fmt"
  21. "runtime/debug"
  22. "sort"
  23. "strings"
  24. "time"
  25. "github.com/88250/lute/parse"
  26. "github.com/dustin/go-humanize"
  27. "github.com/emirpasic/gods/sets/hashset"
  28. "github.com/siyuan-note/eventbus"
  29. "github.com/siyuan-note/logging"
  30. "github.com/siyuan-note/siyuan/kernel/cache"
  31. "github.com/siyuan-note/siyuan/kernel/filesys"
  32. "github.com/siyuan-note/siyuan/kernel/sql"
  33. "github.com/siyuan-note/siyuan/kernel/treenode"
  34. "github.com/siyuan-note/siyuan/kernel/util"
  35. )
  36. func (box *Box) Index(fullRebuildIndex bool) (treeCount int, treeSize int64) {
  37. defer debug.FreeOSMemory()
  38. sql.IndexMode()
  39. defer sql.NormalMode()
  40. //os.MkdirAll("pprof", 0755)
  41. //cpuProfile, _ := os.Create("pprof/cpu_profile_index")
  42. //pprof.StartCPUProfile(cpuProfile)
  43. //defer pprof.StopCPUProfile()
  44. util.SetBootDetails("Listing files...")
  45. files := box.ListFiles("/")
  46. boxLen := len(Conf.GetOpenedBoxes())
  47. if 1 > boxLen {
  48. boxLen = 1
  49. }
  50. bootProgressPart := 10.0 / float64(boxLen) / float64(len(files))
  51. luteEngine := NewLute()
  52. idTitleMap := map[string]string{}
  53. idHashMap := map[string]string{}
  54. util.PushEndlessProgress(fmt.Sprintf("["+box.Name+"] "+Conf.Language(64), len(files)))
  55. i := 0
  56. // 读取并缓存路径映射
  57. for _, file := range files {
  58. if file.isdir || !strings.HasSuffix(file.name, ".sy") {
  59. continue
  60. }
  61. p := file.path
  62. tree, err := filesys.LoadTree(box.ID, p, luteEngine)
  63. if nil != err {
  64. logging.LogErrorf("read box [%s] tree [%s] failed: %s", box.ID, p, err)
  65. continue
  66. }
  67. docIAL := parse.IAL2MapUnEsc(tree.Root.KramdownIAL)
  68. cache.PutDocIAL(p, docIAL)
  69. util.IncBootProgress(bootProgressPart, fmt.Sprintf(Conf.Language(92), util.ShortPathForBootingDisplay(tree.Path)))
  70. treeSize += file.size
  71. treeCount++
  72. // 缓存文档标题,后面做 Path -> HPath 路径映射时需要
  73. idTitleMap[tree.ID] = tree.Root.IALAttr("title")
  74. // 缓存块树
  75. treenode.IndexBlockTree(tree)
  76. // 缓存 ID-Hash,后面需要用于判断是否要重建库
  77. idHashMap[tree.ID] = tree.Hash
  78. if 1 < i && 0 == i%64 {
  79. util.PushEndlessProgress(fmt.Sprintf(Conf.Language(88), i, len(files)-i))
  80. }
  81. i++
  82. }
  83. box.UpdateHistoryGenerated() // 初始化历史生成时间为当前时间
  84. // 检查是否需要重新建库
  85. util.SetBootDetails("Checking data hashes...")
  86. var ids []string
  87. for id := range idTitleMap {
  88. ids = append(ids, id)
  89. }
  90. sort.Slice(ids, func(i, j int) bool { return ids[i] >= ids[j] })
  91. buf := bytes.Buffer{}
  92. for _, id := range ids {
  93. hash, _ := idHashMap[id]
  94. buf.WriteString(hash)
  95. util.SetBootDetails("Checking hash " + hash)
  96. }
  97. boxHash := fmt.Sprintf("%x", sha256.Sum256(buf.Bytes()))
  98. dbBoxHash := sql.GetBoxHash(box.ID)
  99. if boxHash == dbBoxHash {
  100. //logging.LogInfof("use existing database for box [%s]", box.ID)
  101. util.SetBootDetails("Use existing database for notebook " + box.ID)
  102. return
  103. }
  104. // 开始重建库
  105. sql.DisableCache()
  106. defer sql.EnableCache()
  107. start := time.Now()
  108. if !fullRebuildIndex {
  109. tx, err := sql.BeginTx()
  110. if nil != err {
  111. return
  112. }
  113. sql.PutBoxHash(tx, box.ID, boxHash)
  114. util.SetBootDetails("Cleaning obsolete indexes...")
  115. util.PushEndlessProgress(Conf.Language(108))
  116. sql.DeleteByBoxTx(tx, box.ID)
  117. if err = sql.CommitTx(tx); nil != err {
  118. return
  119. }
  120. }
  121. bootProgressPart = 20.0 / float64(boxLen) / float64(treeCount)
  122. context := map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBarAndProgress}
  123. i = 0
  124. // 块级行级入库,缓存块
  125. // 这里不能并行插入,因为 SQLite 不支持
  126. for _, file := range files {
  127. if file.isdir || !strings.HasSuffix(file.name, ".sy") {
  128. continue
  129. }
  130. tree, err := filesys.LoadTree(box.ID, file.path, luteEngine)
  131. if nil != err {
  132. logging.LogErrorf("read box [%s] tree [%s] failed: %s", box.ID, file.path, err)
  133. continue
  134. }
  135. util.IncBootProgress(bootProgressPart, fmt.Sprintf(Conf.Language(93), util.ShortPathForBootingDisplay(tree.Path)))
  136. tx, err := sql.BeginTx()
  137. if nil != err {
  138. continue
  139. }
  140. if err = sql.InsertBlocksSpans(tx, tree, context); nil != err {
  141. sql.RollbackTx(tx)
  142. continue
  143. }
  144. if err = sql.CommitTx(tx); nil != err {
  145. continue
  146. }
  147. if 1 < i && 0 == i%64 {
  148. util.PushEndlessProgress(fmt.Sprintf("["+box.Name+"] "+Conf.Language(53), i, treeCount-i))
  149. }
  150. i++
  151. }
  152. end := time.Now()
  153. elapsed := end.Sub(start).Seconds()
  154. logging.LogInfof("rebuilt database for notebook [%s] in [%.2fs], tree [count=%d, size=%s]", box.ID, elapsed, treeCount, humanize.Bytes(uint64(treeSize)))
  155. util.PushEndlessProgress(fmt.Sprintf(Conf.Language(56), treeCount))
  156. return
  157. }
  158. func IndexRefs() {
  159. sql.EnableCache()
  160. defer sql.ClearBlockCache()
  161. start := time.Now()
  162. util.SetBootDetails("Resolving refs...")
  163. util.PushEndlessProgress(Conf.Language(54))
  164. // 引用入库
  165. util.SetBootDetails("Indexing refs...")
  166. refBlocks := sql.GetRefExistedBlocks()
  167. refTreeIDs := hashset.New()
  168. for _, refBlock := range refBlocks {
  169. refTreeIDs.Add(refBlock.RootID)
  170. }
  171. if 0 < refTreeIDs.Size() {
  172. luteEngine := NewLute()
  173. bootProgressPart := 10.0 / float64(refTreeIDs.Size())
  174. for _, box := range Conf.GetOpenedBoxes() {
  175. tx, err := sql.BeginTx()
  176. if nil != err {
  177. return
  178. }
  179. sql.DeleteRefsByBoxTx(tx, box.ID)
  180. sql.CommitTx(tx)
  181. files := box.ListFiles("/")
  182. i := 0
  183. for _, file := range files {
  184. if file.isdir || !strings.HasSuffix(file.name, ".sy") {
  185. continue
  186. }
  187. if file.isdir || !strings.HasSuffix(file.name, ".sy") {
  188. continue
  189. }
  190. id := strings.TrimSuffix(file.name, ".sy")
  191. if !refTreeIDs.Contains(id) {
  192. continue
  193. }
  194. util.IncBootProgress(bootProgressPart, "Indexing ref "+util.ShortPathForBootingDisplay(file.path))
  195. tree, err := filesys.LoadTree(box.ID, file.path, luteEngine)
  196. if nil != err {
  197. logging.LogErrorf("parse box [%s] tree [%s] failed", box.ID, file.path)
  198. continue
  199. }
  200. tx, err = sql.BeginTx()
  201. if nil != err {
  202. continue
  203. }
  204. sql.InsertRefs(tx, tree)
  205. if err = sql.CommitTx(tx); nil != err {
  206. continue
  207. }
  208. if 1 < i && 0 == i%64 {
  209. util.PushEndlessProgress(fmt.Sprintf(Conf.Language(55), i))
  210. }
  211. i++
  212. }
  213. }
  214. }
  215. logging.LogInfof("resolved refs [%d] in [%dms]", len(refBlocks), time.Now().Sub(start).Milliseconds())
  216. }
  217. func init() {
  218. eventbus.Subscribe(eventbus.EvtSQLInsertBlocks, func(context map[string]interface{}, blockCount int, hash string) {
  219. if util.ContainerAndroid == util.Container || util.ContainerIOS == util.Container {
  220. // Android/iOS 端不显示数据索引和搜索索引状态提示 https://github.com/siyuan-note/siyuan/issues/6392
  221. return
  222. }
  223. msg := fmt.Sprintf(Conf.Language(89), blockCount, hash)
  224. util.SetBootDetails(msg)
  225. util.ContextPushMsg(context, msg)
  226. })
  227. eventbus.Subscribe(eventbus.EvtSQLInsertBlocksFTS, func(context map[string]interface{}, blockCount int, hash string) {
  228. if util.ContainerAndroid == util.Container || util.ContainerIOS == util.Container {
  229. // Android/iOS 端不显示数据索引和搜索索引状态提示 https://github.com/siyuan-note/siyuan/issues/6392
  230. return
  231. }
  232. msg := fmt.Sprintf(Conf.Language(90), blockCount, hash)
  233. util.SetBootDetails(msg)
  234. util.ContextPushMsg(context, msg)
  235. })
  236. }