box.go 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544
  1. // SiYuan - Refactor your thinking
  2. // Copyright (c) 2020-present, b3log.org
  3. //
  4. // This program is free software: you can redistribute it and/or modify
  5. // it under the terms of the GNU Affero General Public License as published by
  6. // the Free Software Foundation, either version 3 of the License, or
  7. // (at your option) any later version.
  8. //
  9. // This program is distributed in the hope that it will be useful,
  10. // but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12. // GNU Affero General Public License for more details.
  13. //
  14. // You should have received a copy of the GNU Affero General Public License
  15. // along with this program. If not, see <https://www.gnu.org/licenses/>.
  16. package model
  17. import (
  18. "bytes"
  19. "errors"
  20. "fmt"
  21. "os"
  22. "path"
  23. "path/filepath"
  24. "runtime/debug"
  25. "sort"
  26. "strings"
  27. "sync"
  28. "time"
  29. "github.com/88250/gulu"
  30. "github.com/88250/lute/ast"
  31. "github.com/88250/lute/html"
  32. "github.com/88250/lute/parse"
  33. "github.com/dustin/go-humanize"
  34. "github.com/facette/natsort"
  35. "github.com/siyuan-note/filelock"
  36. "github.com/siyuan-note/logging"
  37. "github.com/siyuan-note/siyuan/kernel/conf"
  38. "github.com/siyuan-note/siyuan/kernel/filesys"
  39. "github.com/siyuan-note/siyuan/kernel/sql"
  40. "github.com/siyuan-note/siyuan/kernel/task"
  41. "github.com/siyuan-note/siyuan/kernel/treenode"
  42. "github.com/siyuan-note/siyuan/kernel/util"
  43. )
  44. // Box 笔记本。
  45. type Box struct {
  46. ID string `json:"id"`
  47. Name string `json:"name"`
  48. Icon string `json:"icon"`
  49. Sort int `json:"sort"`
  50. SortMode int `json:"sortMode"`
  51. Closed bool `json:"closed"`
  52. NewFlashcardCount int `json:"newFlashcardCount"`
  53. DueFlashcardCount int `json:"dueFlashcardCount"`
  54. FlashcardCount int `json:"flashcardCount"`
  55. historyGenerated int64 // 最近一次历史生成时间
  56. }
  57. var statLock = sync.Mutex{}
  58. func StatJob() {
  59. statLock.Lock()
  60. defer statLock.Unlock()
  61. Conf.Stat.TreeCount = treenode.CountTrees()
  62. Conf.Stat.CTreeCount = treenode.CeilTreeCount(Conf.Stat.TreeCount)
  63. Conf.Stat.BlockCount = treenode.CountBlocks()
  64. Conf.Stat.CBlockCount = treenode.CeilBlockCount(Conf.Stat.BlockCount)
  65. Conf.Stat.DataSize, Conf.Stat.AssetsSize = util.DataSize()
  66. Conf.Stat.CDataSize = util.CeilSize(Conf.Stat.DataSize)
  67. Conf.Stat.CAssetsSize = util.CeilSize(Conf.Stat.AssetsSize)
  68. Conf.Save()
  69. logging.LogInfof("auto stat [trees=%d, blocks=%d, dataSize=%s, assetsSize=%s]", Conf.Stat.TreeCount, Conf.Stat.BlockCount, humanize.Bytes(uint64(Conf.Stat.DataSize)), humanize.Bytes(uint64(Conf.Stat.AssetsSize)))
  70. // 桌面端检查磁盘可用空间 https://github.com/siyuan-note/siyuan/issues/6873
  71. if util.ContainerStd != util.Container {
  72. return
  73. }
  74. if util.NeedWarnDiskUsage(Conf.Stat.DataSize) {
  75. util.PushMsg(Conf.Language(179), 7000)
  76. }
  77. }
  78. func ListNotebooks() (ret []*Box, err error) {
  79. ret = []*Box{}
  80. dirs, err := os.ReadDir(util.DataDir)
  81. if nil != err {
  82. logging.LogErrorf("read dir [%s] failed: %s", util.DataDir, err)
  83. return ret, err
  84. }
  85. for _, dir := range dirs {
  86. if util.IsReservedFilename(dir.Name()) {
  87. continue
  88. }
  89. if !dir.IsDir() {
  90. continue
  91. }
  92. if !ast.IsNodeIDPattern(dir.Name()) {
  93. continue
  94. }
  95. boxConf := conf.NewBoxConf()
  96. boxDirPath := filepath.Join(util.DataDir, dir.Name())
  97. boxConfPath := filepath.Join(boxDirPath, ".siyuan", "conf.json")
  98. if !gulu.File.IsExist(boxConfPath) {
  99. logging.LogWarnf("found a corrupted box [%s]", boxDirPath)
  100. continue
  101. }
  102. data, readErr := filelock.ReadFile(boxConfPath)
  103. if nil != readErr {
  104. logging.LogErrorf("read box conf [%s] failed: %s", boxConfPath, readErr)
  105. continue
  106. }
  107. if readErr = gulu.JSON.UnmarshalJSON(data, boxConf); nil != readErr {
  108. logging.LogErrorf("parse box conf [%s] failed: %s", boxConfPath, readErr)
  109. os.RemoveAll(boxConfPath)
  110. continue
  111. }
  112. id := dir.Name()
  113. ret = append(ret, &Box{
  114. ID: id,
  115. Name: boxConf.Name,
  116. Icon: boxConf.Icon,
  117. Sort: boxConf.Sort,
  118. SortMode: boxConf.SortMode,
  119. Closed: boxConf.Closed,
  120. })
  121. }
  122. switch Conf.FileTree.Sort {
  123. case util.SortModeNameASC:
  124. sort.Slice(ret, func(i, j int) bool {
  125. return util.PinYinCompare(util.RemoveEmoji(ret[i].Name), util.RemoveEmoji(ret[j].Name))
  126. })
  127. case util.SortModeNameDESC:
  128. sort.Slice(ret, func(i, j int) bool {
  129. return util.PinYinCompare(util.RemoveEmoji(ret[j].Name), util.RemoveEmoji(ret[i].Name))
  130. })
  131. case util.SortModeUpdatedASC:
  132. case util.SortModeUpdatedDESC:
  133. case util.SortModeAlphanumASC:
  134. sort.Slice(ret, func(i, j int) bool {
  135. return natsort.Compare(util.RemoveEmoji(ret[i].Name), util.RemoveEmoji(ret[j].Name))
  136. })
  137. case util.SortModeAlphanumDESC:
  138. sort.Slice(ret, func(i, j int) bool {
  139. return natsort.Compare(util.RemoveEmoji(ret[j].Name), util.RemoveEmoji(ret[i].Name))
  140. })
  141. case util.SortModeCustom:
  142. sort.Slice(ret, func(i, j int) bool { return ret[i].Sort < ret[j].Sort })
  143. case util.SortModeRefCountASC:
  144. case util.SortModeRefCountDESC:
  145. case util.SortModeCreatedASC:
  146. sort.Slice(ret, func(i, j int) bool { return natsort.Compare(ret[j].ID, ret[i].ID) })
  147. case util.SortModeCreatedDESC:
  148. sort.Slice(ret, func(i, j int) bool { return natsort.Compare(ret[j].ID, ret[i].ID) })
  149. }
  150. return
  151. }
  152. func (box *Box) GetConf() (ret *conf.BoxConf) {
  153. ret = conf.NewBoxConf()
  154. confPath := filepath.Join(util.DataDir, box.ID, ".siyuan/conf.json")
  155. if !gulu.File.IsExist(confPath) {
  156. return
  157. }
  158. data, err := filelock.ReadFile(confPath)
  159. if nil != err {
  160. logging.LogErrorf("read box conf [%s] failed: %s", confPath, err)
  161. return
  162. }
  163. if err = gulu.JSON.UnmarshalJSON(data, ret); nil != err {
  164. logging.LogErrorf("parse box conf [%s] failed: %s", confPath, err)
  165. return
  166. }
  167. return
  168. }
  169. func (box *Box) SaveConf(conf *conf.BoxConf) {
  170. confPath := filepath.Join(util.DataDir, box.ID, ".siyuan/conf.json")
  171. newData, err := gulu.JSON.MarshalIndentJSON(conf, "", " ")
  172. if nil != err {
  173. logging.LogErrorf("marshal box conf [%s] failed: %s", confPath, err)
  174. return
  175. }
  176. oldData, err := filelock.ReadFile(confPath)
  177. if nil != err {
  178. box.saveConf0(newData)
  179. return
  180. }
  181. if bytes.Equal(newData, oldData) {
  182. return
  183. }
  184. box.saveConf0(newData)
  185. }
  186. func (box *Box) saveConf0(data []byte) {
  187. confPath := filepath.Join(util.DataDir, box.ID, ".siyuan/conf.json")
  188. if err := os.MkdirAll(filepath.Join(util.DataDir, box.ID, ".siyuan"), 0755); nil != err {
  189. logging.LogErrorf("save box conf [%s] failed: %s", confPath, err)
  190. }
  191. if err := filelock.WriteFile(confPath, data); nil != err {
  192. logging.LogErrorf("write box conf [%s] failed: %s", confPath, err)
  193. util.ReportFileSysFatalError(err)
  194. return
  195. }
  196. }
  197. func (box *Box) Ls(p string) (ret []*FileInfo, totals int, err error) {
  198. boxLocalPath := filepath.Join(util.DataDir, box.ID)
  199. if strings.HasSuffix(p, ".sy") {
  200. dir := strings.TrimSuffix(p, ".sy")
  201. absDir := filepath.Join(boxLocalPath, dir)
  202. if gulu.File.IsDir(absDir) {
  203. p = dir
  204. } else {
  205. return
  206. }
  207. }
  208. entries, err := os.ReadDir(filepath.Join(util.DataDir, box.ID, p))
  209. if nil != err {
  210. return
  211. }
  212. for _, f := range entries {
  213. info, infoErr := f.Info()
  214. if nil != infoErr {
  215. logging.LogErrorf("read file info failed: %s", infoErr)
  216. continue
  217. }
  218. name := f.Name()
  219. if util.IsReservedFilename(name) {
  220. continue
  221. }
  222. if strings.HasSuffix(name, ".tmp") {
  223. // 移除写入失败时产生的临时文件
  224. removePath := filepath.Join(util.DataDir, box.ID, p, name)
  225. if removeErr := os.Remove(removePath); nil != removeErr {
  226. logging.LogWarnf("remove tmp file [%s] failed: %s", removePath, removeErr)
  227. }
  228. continue
  229. }
  230. totals += 1
  231. fi := &FileInfo{}
  232. fi.name = name
  233. fi.isdir = f.IsDir()
  234. fi.size = info.Size()
  235. fPath := path.Join(p, name)
  236. if f.IsDir() {
  237. fPath += "/"
  238. }
  239. fi.path = fPath
  240. ret = append(ret, fi)
  241. }
  242. return
  243. }
  244. func (box *Box) Stat(p string) (ret *FileInfo) {
  245. absPath := filepath.Join(util.DataDir, box.ID, p)
  246. info, err := os.Stat(absPath)
  247. if nil != err {
  248. if !os.IsNotExist(err) {
  249. logging.LogErrorf("stat [%s] failed: %s", absPath, err)
  250. }
  251. return
  252. }
  253. ret = &FileInfo{
  254. path: p,
  255. name: info.Name(),
  256. size: info.Size(),
  257. isdir: info.IsDir(),
  258. }
  259. return
  260. }
  261. func (box *Box) Exist(p string) bool {
  262. return gulu.File.IsExist(filepath.Join(util.DataDir, box.ID, p))
  263. }
  264. func (box *Box) Mkdir(path string) error {
  265. if err := os.Mkdir(filepath.Join(util.DataDir, box.ID, path), 0755); nil != err {
  266. msg := fmt.Sprintf(Conf.Language(6), box.Name, path, err)
  267. logging.LogErrorf("mkdir [path=%s] in box [%s] failed: %s", path, box.ID, err)
  268. return errors.New(msg)
  269. }
  270. IncSync()
  271. return nil
  272. }
  273. func (box *Box) MkdirAll(path string) error {
  274. if err := os.MkdirAll(filepath.Join(util.DataDir, box.ID, path), 0755); nil != err {
  275. msg := fmt.Sprintf(Conf.Language(6), box.Name, path, err)
  276. logging.LogErrorf("mkdir all [path=%s] in box [%s] failed: %s", path, box.ID, err)
  277. return errors.New(msg)
  278. }
  279. IncSync()
  280. return nil
  281. }
  282. func (box *Box) Move(oldPath, newPath string) error {
  283. boxLocalPath := filepath.Join(util.DataDir, box.ID)
  284. fromPath := filepath.Join(boxLocalPath, oldPath)
  285. toPath := filepath.Join(boxLocalPath, newPath)
  286. if err := filelock.Move(fromPath, toPath); nil != err {
  287. msg := fmt.Sprintf(Conf.Language(5), box.Name, fromPath, err)
  288. logging.LogErrorf("move [path=%s] in box [%s] failed: %s", fromPath, box.Name, err)
  289. return errors.New(msg)
  290. }
  291. if oldDir := path.Dir(oldPath); ast.IsNodeIDPattern(path.Base(oldDir)) {
  292. fromDir := filepath.Join(boxLocalPath, oldDir)
  293. if util.IsEmptyDir(fromDir) {
  294. filelock.Remove(fromDir)
  295. }
  296. }
  297. IncSync()
  298. return nil
  299. }
  300. func (box *Box) Remove(path string) error {
  301. boxLocalPath := filepath.Join(util.DataDir, box.ID)
  302. filePath := filepath.Join(boxLocalPath, path)
  303. if err := filelock.Remove(filePath); nil != err {
  304. msg := fmt.Sprintf(Conf.Language(7), box.Name, path, err)
  305. logging.LogErrorf("remove [path=%s] in box [%s] failed: %s", path, box.ID, err)
  306. return errors.New(msg)
  307. }
  308. IncSync()
  309. return nil
  310. }
  311. func (box *Box) ListFiles(path string) (ret []*FileInfo) {
  312. fis, _, err := box.Ls(path)
  313. if nil != err {
  314. return
  315. }
  316. box.listFiles(&fis, &ret)
  317. return
  318. }
  319. func (box *Box) listFiles(files, ret *[]*FileInfo) {
  320. for _, file := range *files {
  321. if file.isdir {
  322. fis, _, err := box.Ls(file.path)
  323. if nil == err {
  324. box.listFiles(&fis, ret)
  325. }
  326. *ret = append(*ret, file)
  327. } else {
  328. *ret = append(*ret, file)
  329. }
  330. }
  331. return
  332. }
  333. func isSkipFile(filename string) bool {
  334. return strings.HasPrefix(filename, ".") || "node_modules" == filename || "dist" == filename || "target" == filename
  335. }
  336. func moveTree(tree *parse.Tree) {
  337. treenode.SetBlockTreePath(tree)
  338. sql.UpsertTreeQueue(tree)
  339. box := Conf.Box(tree.Box)
  340. box.renameSubTrees(tree)
  341. }
  342. func (box *Box) renameSubTrees(tree *parse.Tree) {
  343. subFiles := box.ListFiles(tree.Path)
  344. box.moveTrees0(subFiles)
  345. }
  346. func (box *Box) moveTrees0(files []*FileInfo) {
  347. luteEngine := util.NewLute()
  348. for _, subFile := range files {
  349. if !strings.HasSuffix(subFile.path, ".sy") {
  350. continue
  351. }
  352. subTree, err := filesys.LoadTree(box.ID, subFile.path, luteEngine) // LoadTree 会重新构造 HPath
  353. if nil != err {
  354. continue
  355. }
  356. treenode.SetBlockTreePath(subTree)
  357. sql.RenameSubTreeQueue(subTree)
  358. msg := fmt.Sprintf(Conf.Language(107), html.EscapeString(subTree.HPath))
  359. util.PushStatusBar(msg)
  360. }
  361. }
  362. func parseKTree(kramdown []byte) (ret *parse.Tree) {
  363. luteEngine := NewLute()
  364. ret = parse.Parse("", kramdown, luteEngine.ParseOptions)
  365. genTreeID(ret)
  366. return
  367. }
  368. func genTreeID(tree *parse.Tree) {
  369. if nil == tree.Root.FirstChild {
  370. tree.Root.AppendChild(treenode.NewParagraph())
  371. }
  372. ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
  373. if !entering {
  374. return ast.WalkContinue
  375. }
  376. if n.IsEmptyBlockIAL() {
  377. // 空段落保留
  378. p := &ast.Node{Type: ast.NodeParagraph}
  379. p.KramdownIAL = parse.Tokens2IAL(n.Tokens)
  380. p.ID = p.IALAttr("id")
  381. n.InsertBefore(p)
  382. return ast.WalkContinue
  383. }
  384. id := n.IALAttr("id")
  385. if "" == id {
  386. n.SetIALAttr("id", n.ID)
  387. }
  388. if "" == n.IALAttr("id") && (ast.NodeParagraph == n.Type || ast.NodeList == n.Type || ast.NodeListItem == n.Type || ast.NodeBlockquote == n.Type ||
  389. ast.NodeMathBlock == n.Type || ast.NodeCodeBlock == n.Type || ast.NodeHeading == n.Type || ast.NodeTable == n.Type || ast.NodeThematicBreak == n.Type ||
  390. ast.NodeYamlFrontMatter == n.Type || ast.NodeBlockQueryEmbed == n.Type || ast.NodeSuperBlock == n.Type ||
  391. ast.NodeHTMLBlock == n.Type || ast.NodeIFrame == n.Type || ast.NodeWidget == n.Type || ast.NodeAudio == n.Type || ast.NodeVideo == n.Type) {
  392. n.ID = ast.NewNodeID()
  393. n.KramdownIAL = [][]string{{"id", n.ID}}
  394. n.InsertAfter(&ast.Node{Type: ast.NodeKramdownBlockIAL, Tokens: []byte("{: id=\"" + n.ID + "\"}")})
  395. n.SetIALAttr("updated", util.TimeFromID(n.ID))
  396. }
  397. if "" == n.ID && 0 < len(n.KramdownIAL) && ast.NodeDocument != n.Type {
  398. n.ID = n.IALAttr("id")
  399. }
  400. if ast.NodeHTMLBlock == n.Type {
  401. tokens := bytes.TrimSpace(n.Tokens)
  402. if !bytes.HasPrefix(tokens, []byte("<div>")) {
  403. tokens = []byte("<div>\n" + string(tokens))
  404. }
  405. if !bytes.HasSuffix(tokens, []byte("</div>")) {
  406. tokens = append(tokens, []byte("\n</div>")...)
  407. }
  408. n.Tokens = tokens
  409. return ast.WalkContinue
  410. }
  411. if ast.NodeInlineHTML == n.Type {
  412. n.Type = ast.NodeText
  413. return ast.WalkContinue
  414. }
  415. if ast.NodeParagraph == n.Type && nil != n.FirstChild && ast.NodeTaskListItemMarker == n.FirstChild.Type {
  416. // 踢掉任务列表的第一个子节点左侧空格
  417. n.FirstChild.Next.Tokens = bytes.TrimLeft(n.FirstChild.Next.Tokens, " ")
  418. // 调整 li.p.tlim 为 li.tlim.p
  419. n.InsertBefore(n.FirstChild)
  420. }
  421. return ast.WalkContinue
  422. })
  423. tree.Root.KramdownIAL = parse.Tokens2IAL(tree.Root.LastChild.Tokens)
  424. return
  425. }
  426. func FullReindex() {
  427. task.AppendTask(task.DatabaseIndexFull, fullReindex)
  428. task.AppendTask(task.DatabaseIndexRef, IndexRefs)
  429. task.AppendTask(task.ReloadUI, util.ReloadUI)
  430. // TODO ReindexAssetContent()
  431. }
  432. func fullReindex() {
  433. util.PushMsg(Conf.Language(35), 7*1000)
  434. WaitForWritingFiles()
  435. if err := sql.InitDatabase(true); nil != err {
  436. os.Exit(logging.ExitCodeReadOnlyDatabase)
  437. return
  438. }
  439. treenode.InitBlockTree(true)
  440. openedBoxes := Conf.GetOpenedBoxes()
  441. for _, openedBox := range openedBoxes {
  442. index(openedBox.ID)
  443. }
  444. treenode.SaveBlockTree(true)
  445. LoadFlashcards()
  446. debug.FreeOSMemory()
  447. }
  448. func ChangeBoxSort(boxIDs []string) {
  449. for i, boxID := range boxIDs {
  450. box := &Box{ID: boxID}
  451. boxConf := box.GetConf()
  452. boxConf.Sort = i + 1
  453. box.SaveConf(boxConf)
  454. }
  455. }
  456. func SetBoxIcon(boxID, icon string) {
  457. box := &Box{ID: boxID}
  458. boxConf := box.GetConf()
  459. boxConf.Icon = icon
  460. box.SaveConf(boxConf)
  461. }
  462. func (box *Box) UpdateHistoryGenerated() {
  463. boxLatestHistoryTime[box.ID] = time.Now()
  464. }
  465. func getBoxesByPaths(paths []string) (ret map[string]*Box) {
  466. ret = map[string]*Box{}
  467. for _, p := range paths {
  468. id := strings.TrimSuffix(path.Base(p), ".sy")
  469. bt := treenode.GetBlockTree(id)
  470. if nil != bt {
  471. ret[p] = Conf.Box(bt.BoxID)
  472. }
  473. }
  474. return
  475. }