541 lines
16 KiB
Go
541 lines
16 KiB
Go
// SiYuan - Refactor your thinking
|
||
// Copyright (c) 2020-present, b3log.org
|
||
//
|
||
// This program is free software: you can redistribute it and/or modify
|
||
// it under the terms of the GNU Affero General Public License as published by
|
||
// the Free Software Foundation, either version 3 of the License, or
|
||
// (at your option) any later version.
|
||
//
|
||
// This program is distributed in the hope that it will be useful,
|
||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||
// GNU Affero General Public License for more details.
|
||
//
|
||
// You should have received a copy of the GNU Affero General Public License
|
||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||
|
||
package sql
|
||
|
||
import (
|
||
"bytes"
|
||
"crypto/sha256"
|
||
"database/sql"
|
||
"fmt"
|
||
"os"
|
||
"path"
|
||
"path/filepath"
|
||
"strings"
|
||
"sync"
|
||
|
||
"github.com/88250/gulu"
|
||
"github.com/88250/lute/parse"
|
||
"github.com/emirpasic/gods/sets/hashset"
|
||
ignore "github.com/sabhiram/go-gitignore"
|
||
"github.com/siyuan-note/eventbus"
|
||
"github.com/siyuan-note/logging"
|
||
"github.com/siyuan-note/siyuan/kernel/util"
|
||
)
|
||
|
||
var luteEngine = util.NewLute()
|
||
|
||
func init() {
|
||
luteEngine.RenderOptions.KramdownBlockIAL = false // 数据库 markdown 字段为标准 md，但是要保留 span block ial
|
||
}
|
||
|
||
const (
|
||
BlocksInsert = "INSERT INTO blocks (id, parent_id, root_id, hash, box, path, hpath, name, alias, memo, tag, content, fcontent, markdown, length, type, subtype, ial, sort, created, updated) VALUES %s"
|
||
BlocksFTSInsert = "INSERT INTO blocks_fts (id, parent_id, root_id, hash, box, path, hpath, name, alias, memo, tag, content, fcontent, markdown, length, type, subtype, ial, sort, created, updated) VALUES %s"
|
||
BlocksFTSCaseInsensitiveInsert = "INSERT INTO blocks_fts_case_insensitive (id, parent_id, root_id, hash, box, path, hpath, name, alias, memo, tag, content, fcontent, markdown, length, type, subtype, ial, sort, created, updated) VALUES %s"
|
||
BlocksPlaceholder = "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
|
||
|
||
SpansInsert = "INSERT INTO spans (id, block_id, root_id, box, path, content, markdown, type, ial) VALUES %s"
|
||
SpansPlaceholder = "(?, ?, ?, ?, ?, ?, ?, ?, ?)"
|
||
|
||
AssetsPlaceholder = "(?, ?, ?, ?, ?, ?, ?, ?, ?)"
|
||
AttributesPlaceholder = "(?, ?, ?, ?, ?, ?, ?, ?)"
|
||
RefsPlaceholder = "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
|
||
FileAnnotationRefsPlaceholder = "(?, ?, ?, ?, ?, ?, ?, ?, ?)"
|
||
)
|
||
|
||
func insertBlocks(tx *sql.Tx, blocks []*Block, context map[string]interface{}) (err error) {
|
||
if 1 > len(blocks) {
|
||
return
|
||
}
|
||
|
||
var bulk []*Block
|
||
for _, block := range blocks {
|
||
bulk = append(bulk, block)
|
||
if 512 > len(bulk) {
|
||
continue
|
||
}
|
||
|
||
if err = insertBlocks0(tx, bulk, context); err != nil {
|
||
return
|
||
}
|
||
bulk = []*Block{}
|
||
}
|
||
if 0 < len(bulk) {
|
||
if err = insertBlocks0(tx, bulk, context); err != nil {
|
||
return
|
||
}
|
||
}
|
||
return
|
||
}
|
||
|
||
func insertBlocks0(tx *sql.Tx, bulk []*Block, context map[string]interface{}) (err error) {
|
||
valueStrings := make([]string, 0, len(bulk))
|
||
valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(BlocksPlaceholder, "?"))
|
||
hashBuf := bytes.Buffer{}
|
||
for _, b := range bulk {
|
||
valueStrings = append(valueStrings, BlocksPlaceholder)
|
||
valueArgs = append(valueArgs, b.ID)
|
||
valueArgs = append(valueArgs, b.ParentID)
|
||
valueArgs = append(valueArgs, b.RootID)
|
||
valueArgs = append(valueArgs, b.Hash)
|
||
valueArgs = append(valueArgs, b.Box)
|
||
valueArgs = append(valueArgs, b.Path)
|
||
valueArgs = append(valueArgs, b.HPath)
|
||
valueArgs = append(valueArgs, b.Name)
|
||
valueArgs = append(valueArgs, b.Alias)
|
||
valueArgs = append(valueArgs, b.Memo)
|
||
valueArgs = append(valueArgs, b.Tag)
|
||
valueArgs = append(valueArgs, b.Content)
|
||
valueArgs = append(valueArgs, b.FContent)
|
||
valueArgs = append(valueArgs, b.Markdown)
|
||
valueArgs = append(valueArgs, b.Length)
|
||
valueArgs = append(valueArgs, b.Type)
|
||
valueArgs = append(valueArgs, b.SubType)
|
||
valueArgs = append(valueArgs, b.IAL)
|
||
valueArgs = append(valueArgs, b.Sort)
|
||
valueArgs = append(valueArgs, b.Created)
|
||
valueArgs = append(valueArgs, b.Updated)
|
||
putBlockCache(b)
|
||
|
||
hashBuf.WriteString(b.Hash)
|
||
}
|
||
|
||
stmt := fmt.Sprintf(BlocksInsert, strings.Join(valueStrings, ","))
|
||
if err = prepareExecInsertTx(tx, stmt, valueArgs); err != nil {
|
||
return
|
||
}
|
||
hashBuf.WriteString("blocks")
|
||
evtHash := fmt.Sprintf("%x", sha256.Sum256(hashBuf.Bytes()))[:7]
|
||
// 使用下面的 EvtSQLInsertBlocksFTS 就可以了
|
||
//eventbus.Publish(eventbus.EvtSQLInsertBlocks, context, current, total, len(bulk), evtHash)
|
||
|
||
stmt = fmt.Sprintf(BlocksFTSInsert, strings.Join(valueStrings, ","))
|
||
if err = prepareExecInsertTx(tx, stmt, valueArgs); err != nil {
|
||
return
|
||
}
|
||
|
||
if !caseSensitive {
|
||
stmt = fmt.Sprintf(BlocksFTSCaseInsensitiveInsert, strings.Join(valueStrings, ","))
|
||
if err = prepareExecInsertTx(tx, stmt, valueArgs); err != nil {
|
||
return
|
||
}
|
||
}
|
||
hashBuf.WriteString("fts")
|
||
evtHash = fmt.Sprintf("%x", sha256.Sum256(hashBuf.Bytes()))[:7]
|
||
eventbus.Publish(eventbus.EvtSQLInsertBlocksFTS, context, len(bulk), evtHash)
|
||
return
|
||
}
|
||
|
||
func insertAttributes(tx *sql.Tx, attributes []*Attribute) (err error) {
|
||
if 1 > len(attributes) {
|
||
return
|
||
}
|
||
|
||
var bulk []*Attribute
|
||
for _, attr := range attributes {
|
||
bulk = append(bulk, attr)
|
||
if 512 > len(bulk) {
|
||
continue
|
||
}
|
||
|
||
if err = insertAttribute0(tx, bulk); err != nil {
|
||
return
|
||
}
|
||
bulk = []*Attribute{}
|
||
}
|
||
if 0 < len(bulk) {
|
||
if err = insertAttribute0(tx, bulk); err != nil {
|
||
return
|
||
}
|
||
}
|
||
return
|
||
}
|
||
|
||
func insertAttribute0(tx *sql.Tx, bulk []*Attribute) (err error) {
|
||
if 1 > len(bulk) {
|
||
return
|
||
}
|
||
|
||
valueStrings := make([]string, 0, len(bulk))
|
||
valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(AttributesPlaceholder, "?"))
|
||
for _, attr := range bulk {
|
||
valueStrings = append(valueStrings, AttributesPlaceholder)
|
||
valueArgs = append(valueArgs, attr.ID)
|
||
valueArgs = append(valueArgs, attr.Name)
|
||
valueArgs = append(valueArgs, attr.Value)
|
||
valueArgs = append(valueArgs, attr.Type)
|
||
valueArgs = append(valueArgs, attr.BlockID)
|
||
valueArgs = append(valueArgs, attr.RootID)
|
||
valueArgs = append(valueArgs, attr.Box)
|
||
valueArgs = append(valueArgs, attr.Path)
|
||
}
|
||
stmt := fmt.Sprintf("INSERT INTO attributes (id, name, value, type, block_id, root_id, box, path) VALUES %s", strings.Join(valueStrings, ","))
|
||
err = prepareExecInsertTx(tx, stmt, valueArgs)
|
||
return
|
||
}
|
||
|
||
func insertAssets(tx *sql.Tx, assets []*Asset) (err error) {
|
||
if 1 > len(assets) {
|
||
return
|
||
}
|
||
|
||
var bulk []*Asset
|
||
for _, asset := range assets {
|
||
bulk = append(bulk, asset)
|
||
if 512 > len(bulk) {
|
||
continue
|
||
}
|
||
|
||
if err = insertAsset0(tx, bulk); err != nil {
|
||
return
|
||
}
|
||
bulk = []*Asset{}
|
||
}
|
||
if 0 < len(bulk) {
|
||
if err = insertAsset0(tx, bulk); err != nil {
|
||
return
|
||
}
|
||
}
|
||
return
|
||
}
|
||
|
||
func insertAsset0(tx *sql.Tx, bulk []*Asset) (err error) {
|
||
if 1 > len(bulk) {
|
||
return
|
||
}
|
||
|
||
valueStrings := make([]string, 0, len(bulk))
|
||
valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(AssetsPlaceholder, "?"))
|
||
for _, asset := range bulk {
|
||
valueStrings = append(valueStrings, AssetsPlaceholder)
|
||
valueArgs = append(valueArgs, asset.ID)
|
||
valueArgs = append(valueArgs, asset.BlockID)
|
||
valueArgs = append(valueArgs, asset.RootID)
|
||
valueArgs = append(valueArgs, asset.Box)
|
||
valueArgs = append(valueArgs, asset.DocPath)
|
||
valueArgs = append(valueArgs, asset.Path)
|
||
valueArgs = append(valueArgs, asset.Name)
|
||
valueArgs = append(valueArgs, asset.Title)
|
||
valueArgs = append(valueArgs, asset.Hash)
|
||
}
|
||
stmt := fmt.Sprintf("INSERT INTO assets (id, block_id, root_id, box, docpath, path, name, title, hash) VALUES %s", strings.Join(valueStrings, ","))
|
||
err = prepareExecInsertTx(tx, stmt, valueArgs)
|
||
return
|
||
}
|
||
|
||
func insertSpans(tx *sql.Tx, spans []*Span) (err error) {
|
||
if 1 > len(spans) {
|
||
return
|
||
}
|
||
|
||
var bulk []*Span
|
||
for _, span := range spans {
|
||
bulk = append(bulk, span)
|
||
if 512 > len(bulk) {
|
||
continue
|
||
}
|
||
|
||
if err = insertSpans0(tx, bulk); err != nil {
|
||
return
|
||
}
|
||
bulk = []*Span{}
|
||
}
|
||
if 0 < len(bulk) {
|
||
if err = insertSpans0(tx, bulk); err != nil {
|
||
return
|
||
}
|
||
}
|
||
return
|
||
}
|
||
|
||
func insertSpans0(tx *sql.Tx, bulk []*Span) (err error) {
|
||
if 1 > len(bulk) {
|
||
return
|
||
}
|
||
|
||
valueStrings := make([]string, 0, len(bulk))
|
||
valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(SpansPlaceholder, "?"))
|
||
for _, span := range bulk {
|
||
valueStrings = append(valueStrings, SpansPlaceholder)
|
||
valueArgs = append(valueArgs, span.ID)
|
||
valueArgs = append(valueArgs, span.BlockID)
|
||
valueArgs = append(valueArgs, span.RootID)
|
||
valueArgs = append(valueArgs, span.Box)
|
||
valueArgs = append(valueArgs, span.Path)
|
||
valueArgs = append(valueArgs, span.Content)
|
||
valueArgs = append(valueArgs, span.Markdown)
|
||
valueArgs = append(valueArgs, span.Type)
|
||
valueArgs = append(valueArgs, span.IAL)
|
||
}
|
||
stmt := fmt.Sprintf(SpansInsert, strings.Join(valueStrings, ","))
|
||
err = prepareExecInsertTx(tx, stmt, valueArgs)
|
||
return
|
||
}
|
||
|
||
func insertBlockRefs(tx *sql.Tx, refs []*Ref) (err error) {
|
||
if 1 > len(refs) {
|
||
return
|
||
}
|
||
|
||
var bulk []*Ref
|
||
for _, ref := range refs {
|
||
bulk = append(bulk, ref)
|
||
if 512 > len(bulk) {
|
||
continue
|
||
}
|
||
|
||
if err = insertRefs0(tx, bulk); err != nil {
|
||
return
|
||
}
|
||
bulk = []*Ref{}
|
||
}
|
||
if 0 < len(bulk) {
|
||
if err = insertRefs0(tx, bulk); err != nil {
|
||
return
|
||
}
|
||
}
|
||
return
|
||
}
|
||
|
||
func insertRefs0(tx *sql.Tx, bulk []*Ref) (err error) {
|
||
if 1 > len(bulk) {
|
||
return
|
||
}
|
||
|
||
valueStrings := make([]string, 0, len(bulk))
|
||
valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(RefsPlaceholder, "?"))
|
||
for _, ref := range bulk {
|
||
valueStrings = append(valueStrings, RefsPlaceholder)
|
||
valueArgs = append(valueArgs, ref.ID)
|
||
valueArgs = append(valueArgs, ref.DefBlockID)
|
||
valueArgs = append(valueArgs, ref.DefBlockParentID)
|
||
valueArgs = append(valueArgs, ref.DefBlockRootID)
|
||
valueArgs = append(valueArgs, ref.DefBlockPath)
|
||
valueArgs = append(valueArgs, ref.BlockID)
|
||
valueArgs = append(valueArgs, ref.RootID)
|
||
valueArgs = append(valueArgs, ref.Box)
|
||
valueArgs = append(valueArgs, ref.Path)
|
||
valueArgs = append(valueArgs, ref.Content)
|
||
valueArgs = append(valueArgs, ref.Markdown)
|
||
valueArgs = append(valueArgs, ref.Type)
|
||
|
||
putRefCache(ref)
|
||
}
|
||
stmt := fmt.Sprintf("INSERT INTO refs (id, def_block_id, def_block_parent_id, def_block_root_id, def_block_path, block_id, root_id, box, path, content, markdown, type) VALUES %s", strings.Join(valueStrings, ","))
|
||
err = prepareExecInsertTx(tx, stmt, valueArgs)
|
||
return
|
||
}
|
||
|
||
func insertFileAnnotationRefs(tx *sql.Tx, refs []*FileAnnotationRef) (err error) {
|
||
if 1 > len(refs) {
|
||
return
|
||
}
|
||
|
||
var bulk []*FileAnnotationRef
|
||
for _, ref := range refs {
|
||
bulk = append(bulk, ref)
|
||
if 512 > len(bulk) {
|
||
continue
|
||
}
|
||
|
||
if err = insertFileAnnotationRefs0(tx, bulk); err != nil {
|
||
return
|
||
}
|
||
bulk = []*FileAnnotationRef{}
|
||
}
|
||
if 0 < len(bulk) {
|
||
if err = insertFileAnnotationRefs0(tx, bulk); err != nil {
|
||
return
|
||
}
|
||
}
|
||
return
|
||
}
|
||
|
||
func insertFileAnnotationRefs0(tx *sql.Tx, bulk []*FileAnnotationRef) (err error) {
|
||
if 1 > len(bulk) {
|
||
return
|
||
}
|
||
|
||
valueStrings := make([]string, 0, len(bulk))
|
||
valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(FileAnnotationRefsPlaceholder, "?"))
|
||
for _, ref := range bulk {
|
||
valueStrings = append(valueStrings, FileAnnotationRefsPlaceholder)
|
||
valueArgs = append(valueArgs, ref.ID)
|
||
valueArgs = append(valueArgs, ref.FilePath)
|
||
valueArgs = append(valueArgs, ref.AnnotationID)
|
||
valueArgs = append(valueArgs, ref.BlockID)
|
||
valueArgs = append(valueArgs, ref.RootID)
|
||
valueArgs = append(valueArgs, ref.Box)
|
||
valueArgs = append(valueArgs, ref.Path)
|
||
valueArgs = append(valueArgs, ref.Content)
|
||
valueArgs = append(valueArgs, ref.Type)
|
||
}
|
||
stmt := fmt.Sprintf("INSERT INTO file_annotation_refs (id, file_path, annotation_id, block_id, root_id, box, path, content, type) VALUES %s", strings.Join(valueStrings, ","))
|
||
err = prepareExecInsertTx(tx, stmt, valueArgs)
|
||
return
|
||
}
|
||
|
||
func indexTree(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}) (err error) {
|
||
blocks, spans, assets, attributes := fromTree(tree.Root, tree)
|
||
refs, fileAnnotationRefs := refsFromTree(tree)
|
||
err = insertTree0(tx, tree, context, blocks, spans, assets, attributes, refs, fileAnnotationRefs)
|
||
return
|
||
}
|
||
|
||
func upsertTree(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}) (err error) {
|
||
oldBlockHashes := queryBlockHashes(tree.ID)
|
||
blocks, spans, assets, attributes := fromTree(tree.Root, tree)
|
||
newBlockHashes := map[string]string{}
|
||
for _, block := range blocks {
|
||
newBlockHashes[block.ID] = block.Hash
|
||
}
|
||
unChanges := hashset.New()
|
||
var toRemoves []string
|
||
for id, hash := range oldBlockHashes {
|
||
if newHash, ok := newBlockHashes[id]; ok {
|
||
if newHash == hash {
|
||
unChanges.Add(id)
|
||
}
|
||
} else {
|
||
toRemoves = append(toRemoves, id)
|
||
}
|
||
}
|
||
tmp := blocks[:0]
|
||
for _, b := range blocks {
|
||
if !unChanges.Contains(b.ID) {
|
||
tmp = append(tmp, b)
|
||
}
|
||
}
|
||
blocks = tmp
|
||
for _, b := range blocks {
|
||
toRemoves = append(toRemoves, b.ID)
|
||
}
|
||
|
||
if err = deleteBlocksByIDs(tx, toRemoves); err != nil {
|
||
return
|
||
}
|
||
|
||
if err = deleteSpansByRootID(tx, tree.ID); err != nil {
|
||
return
|
||
}
|
||
if err = deleteAssetsByRootID(tx, tree.ID); err != nil {
|
||
return
|
||
}
|
||
if err = deleteAttributesByRootID(tx, tree.ID); err != nil {
|
||
return
|
||
}
|
||
if err = deleteRefsByPathTx(tx, tree.Box, tree.Path); err != nil {
|
||
return
|
||
}
|
||
if err = deleteFileAnnotationRefsByPathTx(tx, tree.Box, tree.Path); err != nil {
|
||
return
|
||
}
|
||
|
||
refs, fileAnnotationRefs := refsFromTree(tree)
|
||
if err = insertTree0(tx, tree, context, blocks, spans, assets, attributes, refs, fileAnnotationRefs); err != nil {
|
||
return
|
||
}
|
||
return err
|
||
}
|
||
|
||
func insertTree0(tx *sql.Tx, tree *parse.Tree, context map[string]interface{},
|
||
blocks []*Block, spans []*Span, assets []*Asset, attributes []*Attribute,
|
||
refs []*Ref, fileAnnotationRefs []*FileAnnotationRef) (err error) {
|
||
if ignoreLines := getIndexIgnoreLines(); 0 < len(ignoreLines) {
|
||
// Support ignore index https://github.com/siyuan-note/siyuan/issues/9198
|
||
matcher := ignore.CompileIgnoreLines(ignoreLines...)
|
||
if matcher.MatchesPath("/" + path.Join(tree.Box, tree.Path)) {
|
||
return
|
||
}
|
||
}
|
||
|
||
if err = insertBlocks(tx, blocks, context); err != nil {
|
||
return
|
||
}
|
||
|
||
if err = insertBlockRefs(tx, refs); err != nil {
|
||
return
|
||
}
|
||
if err = insertFileAnnotationRefs(tx, fileAnnotationRefs); err != nil {
|
||
return
|
||
}
|
||
|
||
if 0 < len(spans) {
|
||
// 移除文档标签，否则会重复添加 https://github.com/siyuan-note/siyuan/issues/3723
|
||
if err = deleteSpansByRootID(tx, tree.Root.ID); err != nil {
|
||
return
|
||
}
|
||
if err = insertSpans(tx, spans); err != nil {
|
||
return
|
||
}
|
||
}
|
||
if err = insertAssets(tx, assets); err != nil {
|
||
return
|
||
}
|
||
if err = insertAttributes(tx, attributes); err != nil {
|
||
return
|
||
}
|
||
return
|
||
}
|
||
|
||
var (
|
||
IndexIgnoreCached bool
|
||
indexIgnore []string
|
||
indexIgnoreLock = sync.Mutex{}
|
||
)
|
||
|
||
func getIndexIgnoreLines() (ret []string) {
|
||
// Support ignore index https://github.com/siyuan-note/siyuan/issues/9198
|
||
|
||
if IndexIgnoreCached {
|
||
return indexIgnore
|
||
}
|
||
|
||
indexIgnoreLock.Lock()
|
||
defer indexIgnoreLock.Unlock()
|
||
|
||
IndexIgnoreCached = true
|
||
indexIgnorePath := filepath.Join(util.DataDir, ".siyuan", "indexignore")
|
||
err := os.MkdirAll(filepath.Dir(indexIgnorePath), 0755)
|
||
if err != nil {
|
||
return
|
||
}
|
||
if !gulu.File.IsExist(indexIgnorePath) {
|
||
if err = gulu.File.WriteFileSafer(indexIgnorePath, nil, 0644); err != nil {
|
||
logging.LogErrorf("create indexignore [%s] failed: %s", indexIgnorePath, err)
|
||
return
|
||
}
|
||
}
|
||
data, err := os.ReadFile(indexIgnorePath)
|
||
if err != nil {
|
||
logging.LogErrorf("read indexignore [%s] failed: %s", indexIgnorePath, err)
|
||
return
|
||
}
|
||
dataStr := string(data)
|
||
dataStr = strings.ReplaceAll(dataStr, "\r\n", "\n")
|
||
ret = strings.Split(dataStr, "\n")
|
||
|
||
ret = gulu.Str.RemoveDuplicatedElem(ret)
|
||
if 0 < len(ret) && "" == ret[0] {
|
||
ret = ret[1:]
|
||
}
|
||
indexIgnore = nil
|
||
for _, line := range ret {
|
||
indexIgnore = append(indexIgnore, line)
|
||
}
|
||
return
|
||
}
|