// SiYuan - Refactor your thinking // Copyright (c) 2020-present, b3log.org // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Affero General Public License for more details. // // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . package sql import ( "bytes" "crypto/sha256" "database/sql" "fmt" "os" "path" "path/filepath" "strings" "sync" "github.com/88250/gulu" "github.com/88250/lute/parse" "github.com/emirpasic/gods/sets/hashset" ignore "github.com/sabhiram/go-gitignore" "github.com/siyuan-note/eventbus" "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/util" ) var luteEngine = util.NewLute() func init() { luteEngine.RenderOptions.KramdownBlockIAL = false // 数据库 markdown 字段为标准 md,但是要保留 span block ial } const ( BlocksInsert = "INSERT INTO blocks (id, parent_id, root_id, hash, box, path, hpath, name, alias, memo, tag, content, fcontent, markdown, length, type, subtype, ial, sort, created, updated) VALUES %s" BlocksFTSInsert = "INSERT INTO blocks_fts (id, parent_id, root_id, hash, box, path, hpath, name, alias, memo, tag, content, fcontent, markdown, length, type, subtype, ial, sort, created, updated) VALUES %s" BlocksFTSCaseInsensitiveInsert = "INSERT INTO blocks_fts_case_insensitive (id, parent_id, root_id, hash, box, path, hpath, name, alias, memo, tag, content, fcontent, markdown, length, type, subtype, ial, sort, created, updated) VALUES %s" BlocksPlaceholder = "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" SpansInsert = "INSERT INTO spans (id, block_id, root_id, box, path, content, markdown, type, ial) VALUES %s" SpansPlaceholder = "(?, ?, ?, ?, ?, ?, ?, ?, ?)" AssetsPlaceholder = "(?, ?, ?, ?, ?, ?, ?, ?, ?)" AttributesPlaceholder = "(?, ?, ?, ?, ?, ?, ?, ?)" RefsPlaceholder = "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" FileAnnotationRefsPlaceholder = "(?, ?, ?, ?, ?, ?, ?, ?, ?)" ) func insertBlocks(tx *sql.Tx, blocks []*Block, context map[string]interface{}) (err error) { if 1 > len(blocks) { return } var bulk []*Block for _, block := range blocks { bulk = append(bulk, block) if 512 > len(bulk) { continue } if err = insertBlocks0(tx, bulk, context); err != nil { return } bulk = []*Block{} } if 0 < len(bulk) { if err = insertBlocks0(tx, bulk, context); err != nil { return } } return } func insertBlocks0(tx *sql.Tx, bulk []*Block, context map[string]interface{}) (err error) { valueStrings := make([]string, 0, len(bulk)) valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(BlocksPlaceholder, "?")) hashBuf := bytes.Buffer{} for _, b := range bulk { valueStrings = append(valueStrings, BlocksPlaceholder) valueArgs = append(valueArgs, b.ID) valueArgs = append(valueArgs, b.ParentID) valueArgs = append(valueArgs, b.RootID) valueArgs = append(valueArgs, b.Hash) valueArgs = append(valueArgs, b.Box) valueArgs = append(valueArgs, b.Path) valueArgs = append(valueArgs, b.HPath) valueArgs = append(valueArgs, b.Name) valueArgs = append(valueArgs, b.Alias) valueArgs = append(valueArgs, b.Memo) valueArgs = append(valueArgs, b.Tag) valueArgs = append(valueArgs, b.Content) valueArgs = append(valueArgs, b.FContent) valueArgs = append(valueArgs, b.Markdown) valueArgs = append(valueArgs, b.Length) valueArgs = append(valueArgs, b.Type) valueArgs = append(valueArgs, b.SubType) valueArgs = append(valueArgs, b.IAL) valueArgs = append(valueArgs, b.Sort) valueArgs = append(valueArgs, b.Created) valueArgs = append(valueArgs, b.Updated) putBlockCache(b) hashBuf.WriteString(b.Hash) } stmt := fmt.Sprintf(BlocksInsert, strings.Join(valueStrings, ",")) if err = prepareExecInsertTx(tx, stmt, valueArgs); err != nil { return } hashBuf.WriteString("blocks") evtHash := fmt.Sprintf("%x", sha256.Sum256(hashBuf.Bytes()))[:7] // 使用下面的 EvtSQLInsertBlocksFTS 就可以了 //eventbus.Publish(eventbus.EvtSQLInsertBlocks, context, current, total, len(bulk), evtHash) stmt = fmt.Sprintf(BlocksFTSInsert, strings.Join(valueStrings, ",")) if err = prepareExecInsertTx(tx, stmt, valueArgs); err != nil { return } if !caseSensitive { stmt = fmt.Sprintf(BlocksFTSCaseInsensitiveInsert, strings.Join(valueStrings, ",")) if err = prepareExecInsertTx(tx, stmt, valueArgs); err != nil { return } } hashBuf.WriteString("fts") evtHash = fmt.Sprintf("%x", sha256.Sum256(hashBuf.Bytes()))[:7] eventbus.Publish(eventbus.EvtSQLInsertBlocksFTS, context, len(bulk), evtHash) return } func insertAttributes(tx *sql.Tx, attributes []*Attribute) (err error) { if 1 > len(attributes) { return } var bulk []*Attribute for _, attr := range attributes { bulk = append(bulk, attr) if 512 > len(bulk) { continue } if err = insertAttribute0(tx, bulk); err != nil { return } bulk = []*Attribute{} } if 0 < len(bulk) { if err = insertAttribute0(tx, bulk); err != nil { return } } return } func insertAttribute0(tx *sql.Tx, bulk []*Attribute) (err error) { if 1 > len(bulk) { return } valueStrings := make([]string, 0, len(bulk)) valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(AttributesPlaceholder, "?")) for _, attr := range bulk { valueStrings = append(valueStrings, AttributesPlaceholder) valueArgs = append(valueArgs, attr.ID) valueArgs = append(valueArgs, attr.Name) valueArgs = append(valueArgs, attr.Value) valueArgs = append(valueArgs, attr.Type) valueArgs = append(valueArgs, attr.BlockID) valueArgs = append(valueArgs, attr.RootID) valueArgs = append(valueArgs, attr.Box) valueArgs = append(valueArgs, attr.Path) } stmt := fmt.Sprintf("INSERT INTO attributes (id, name, value, type, block_id, root_id, box, path) VALUES %s", strings.Join(valueStrings, ",")) err = prepareExecInsertTx(tx, stmt, valueArgs) return } func insertAssets(tx *sql.Tx, assets []*Asset) (err error) { if 1 > len(assets) { return } var bulk []*Asset for _, asset := range assets { bulk = append(bulk, asset) if 512 > len(bulk) { continue } if err = insertAsset0(tx, bulk); err != nil { return } bulk = []*Asset{} } if 0 < len(bulk) { if err = insertAsset0(tx, bulk); err != nil { return } } return } func insertAsset0(tx *sql.Tx, bulk []*Asset) (err error) { if 1 > len(bulk) { return } valueStrings := make([]string, 0, len(bulk)) valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(AssetsPlaceholder, "?")) for _, asset := range bulk { valueStrings = append(valueStrings, AssetsPlaceholder) valueArgs = append(valueArgs, asset.ID) valueArgs = append(valueArgs, asset.BlockID) valueArgs = append(valueArgs, asset.RootID) valueArgs = append(valueArgs, asset.Box) valueArgs = append(valueArgs, asset.DocPath) valueArgs = append(valueArgs, asset.Path) valueArgs = append(valueArgs, asset.Name) valueArgs = append(valueArgs, asset.Title) valueArgs = append(valueArgs, asset.Hash) } stmt := fmt.Sprintf("INSERT INTO assets (id, block_id, root_id, box, docpath, path, name, title, hash) VALUES %s", strings.Join(valueStrings, ",")) err = prepareExecInsertTx(tx, stmt, valueArgs) return } func insertSpans(tx *sql.Tx, spans []*Span) (err error) { if 1 > len(spans) { return } var bulk []*Span for _, span := range spans { bulk = append(bulk, span) if 512 > len(bulk) { continue } if err = insertSpans0(tx, bulk); err != nil { return } bulk = []*Span{} } if 0 < len(bulk) { if err = insertSpans0(tx, bulk); err != nil { return } } return } func insertSpans0(tx *sql.Tx, bulk []*Span) (err error) { if 1 > len(bulk) { return } valueStrings := make([]string, 0, len(bulk)) valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(SpansPlaceholder, "?")) for _, span := range bulk { valueStrings = append(valueStrings, SpansPlaceholder) valueArgs = append(valueArgs, span.ID) valueArgs = append(valueArgs, span.BlockID) valueArgs = append(valueArgs, span.RootID) valueArgs = append(valueArgs, span.Box) valueArgs = append(valueArgs, span.Path) valueArgs = append(valueArgs, span.Content) valueArgs = append(valueArgs, span.Markdown) valueArgs = append(valueArgs, span.Type) valueArgs = append(valueArgs, span.IAL) } stmt := fmt.Sprintf(SpansInsert, strings.Join(valueStrings, ",")) err = prepareExecInsertTx(tx, stmt, valueArgs) return } func insertBlockRefs(tx *sql.Tx, refs []*Ref) (err error) { if 1 > len(refs) { return } var bulk []*Ref for _, ref := range refs { bulk = append(bulk, ref) if 512 > len(bulk) { continue } if err = insertRefs0(tx, bulk); err != nil { return } bulk = []*Ref{} } if 0 < len(bulk) { if err = insertRefs0(tx, bulk); err != nil { return } } return } func insertRefs0(tx *sql.Tx, bulk []*Ref) (err error) { if 1 > len(bulk) { return } valueStrings := make([]string, 0, len(bulk)) valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(RefsPlaceholder, "?")) for _, ref := range bulk { valueStrings = append(valueStrings, RefsPlaceholder) valueArgs = append(valueArgs, ref.ID) valueArgs = append(valueArgs, ref.DefBlockID) valueArgs = append(valueArgs, ref.DefBlockParentID) valueArgs = append(valueArgs, ref.DefBlockRootID) valueArgs = append(valueArgs, ref.DefBlockPath) valueArgs = append(valueArgs, ref.BlockID) valueArgs = append(valueArgs, ref.RootID) valueArgs = append(valueArgs, ref.Box) valueArgs = append(valueArgs, ref.Path) valueArgs = append(valueArgs, ref.Content) valueArgs = append(valueArgs, ref.Markdown) valueArgs = append(valueArgs, ref.Type) putRefCache(ref) } stmt := fmt.Sprintf("INSERT INTO refs (id, def_block_id, def_block_parent_id, def_block_root_id, def_block_path, block_id, root_id, box, path, content, markdown, type) VALUES %s", strings.Join(valueStrings, ",")) err = prepareExecInsertTx(tx, stmt, valueArgs) return } func insertFileAnnotationRefs(tx *sql.Tx, refs []*FileAnnotationRef) (err error) { if 1 > len(refs) { return } var bulk []*FileAnnotationRef for _, ref := range refs { bulk = append(bulk, ref) if 512 > len(bulk) { continue } if err = insertFileAnnotationRefs0(tx, bulk); err != nil { return } bulk = []*FileAnnotationRef{} } if 0 < len(bulk) { if err = insertFileAnnotationRefs0(tx, bulk); err != nil { return } } return } func insertFileAnnotationRefs0(tx *sql.Tx, bulk []*FileAnnotationRef) (err error) { if 1 > len(bulk) { return } valueStrings := make([]string, 0, len(bulk)) valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(FileAnnotationRefsPlaceholder, "?")) for _, ref := range bulk { valueStrings = append(valueStrings, FileAnnotationRefsPlaceholder) valueArgs = append(valueArgs, ref.ID) valueArgs = append(valueArgs, ref.FilePath) valueArgs = append(valueArgs, ref.AnnotationID) valueArgs = append(valueArgs, ref.BlockID) valueArgs = append(valueArgs, ref.RootID) valueArgs = append(valueArgs, ref.Box) valueArgs = append(valueArgs, ref.Path) valueArgs = append(valueArgs, ref.Content) valueArgs = append(valueArgs, ref.Type) } stmt := fmt.Sprintf("INSERT INTO file_annotation_refs (id, file_path, annotation_id, block_id, root_id, box, path, content, type) VALUES %s", strings.Join(valueStrings, ",")) err = prepareExecInsertTx(tx, stmt, valueArgs) return } func indexTree(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}) (err error) { blocks, spans, assets, attributes := fromTree(tree.Root, tree) refs, fileAnnotationRefs := refsFromTree(tree) err = insertTree0(tx, tree, context, blocks, spans, assets, attributes, refs, fileAnnotationRefs) return } func upsertTree(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}) (err error) { oldBlockHashes := queryBlockHashes(tree.ID) blocks, spans, assets, attributes := fromTree(tree.Root, tree) newBlockHashes := map[string]string{} for _, block := range blocks { newBlockHashes[block.ID] = block.Hash } unChanges := hashset.New() var toRemoves []string for id, hash := range oldBlockHashes { if newHash, ok := newBlockHashes[id]; ok { if newHash == hash { unChanges.Add(id) } } else { toRemoves = append(toRemoves, id) } } tmp := blocks[:0] for _, b := range blocks { if !unChanges.Contains(b.ID) { tmp = append(tmp, b) } } blocks = tmp for _, b := range blocks { toRemoves = append(toRemoves, b.ID) } if err = deleteBlocksByIDs(tx, toRemoves); err != nil { return } if err = deleteSpansByRootID(tx, tree.ID); err != nil { return } if err = deleteAssetsByRootID(tx, tree.ID); err != nil { return } if err = deleteAttributesByRootID(tx, tree.ID); err != nil { return } if err = deleteRefsByPathTx(tx, tree.Box, tree.Path); err != nil { return } if err = deleteFileAnnotationRefsByPathTx(tx, tree.Box, tree.Path); err != nil { return } refs, fileAnnotationRefs := refsFromTree(tree) if err = insertTree0(tx, tree, context, blocks, spans, assets, attributes, refs, fileAnnotationRefs); err != nil { return } return err } func insertTree0(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}, blocks []*Block, spans []*Span, assets []*Asset, attributes []*Attribute, refs []*Ref, fileAnnotationRefs []*FileAnnotationRef) (err error) { if ignoreLines := getIndexIgnoreLines(); 0 < len(ignoreLines) { // Support ignore index https://github.com/siyuan-note/siyuan/issues/9198 matcher := ignore.CompileIgnoreLines(ignoreLines...) if matcher.MatchesPath("/" + path.Join(tree.Box, tree.Path)) { return } } if err = insertBlocks(tx, blocks, context); err != nil { return } if err = insertBlockRefs(tx, refs); err != nil { return } if err = insertFileAnnotationRefs(tx, fileAnnotationRefs); err != nil { return } if 0 < len(spans) { // 移除文档标签,否则会重复添加 https://github.com/siyuan-note/siyuan/issues/3723 if err = deleteSpansByRootID(tx, tree.Root.ID); err != nil { return } if err = insertSpans(tx, spans); err != nil { return } } if err = insertAssets(tx, assets); err != nil { return } if err = insertAttributes(tx, attributes); err != nil { return } return } var ( IndexIgnoreCached bool indexIgnore []string indexIgnoreLock = sync.Mutex{} ) func getIndexIgnoreLines() (ret []string) { // Support ignore index https://github.com/siyuan-note/siyuan/issues/9198 if IndexIgnoreCached { return indexIgnore } indexIgnoreLock.Lock() defer indexIgnoreLock.Unlock() IndexIgnoreCached = true indexIgnorePath := filepath.Join(util.DataDir, ".siyuan", "indexignore") err := os.MkdirAll(filepath.Dir(indexIgnorePath), 0755) if err != nil { return } if !gulu.File.IsExist(indexIgnorePath) { if err = gulu.File.WriteFileSafer(indexIgnorePath, nil, 0644); err != nil { logging.LogErrorf("create indexignore [%s] failed: %s", indexIgnorePath, err) return } } data, err := os.ReadFile(indexIgnorePath) if err != nil { logging.LogErrorf("read indexignore [%s] failed: %s", indexIgnorePath, err) return } dataStr := string(data) dataStr = strings.ReplaceAll(dataStr, "\r\n", "\n") ret = strings.Split(dataStr, "\n") ret = gulu.Str.RemoveDuplicatedElem(ret) if 0 < len(ret) && "" == ret[0] { ret = ret[1:] } indexIgnore = nil for _, line := range ret { indexIgnore = append(indexIgnore, line) } return }