0ct0pu5/siyuan
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'origin/dev' into dev

Browse source
This commit is contained in:
Vanessa 2022-12-02 22:09:53 +08:00
commit 4c9356985c
3 changed files with 170 additions and 136 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
kernel/api/search.go
View file

@ -220,12 +220,17 @@ func fullTextSearchBlock(c *gin.Context) {
if nil != methodArg {
method = int(methodArg.(float64))
}
orderByArg := arg["orderBy"]
var orderBy int // 0：按块类型（默认），1：按创建时间升序，2：按创建时间降序，3：按更新时间升序，4：按更新时间降序，5：按内容顺序（仅在按文档分组时）
if nil != orderByArg {
orderBy = int(orderByArg.(float64))
}
groupByArg := arg["groupBy"]
var groupBy int // 0：不分组，1：按文档分组
if nil != groupByArg {
groupBy = int(groupByArg.(float64))
}
blocks, matchedBlockCount, matchedRootCount := model.FullTextSearchBlock(query, boxes, paths, types, method, groupBy)
blocks, matchedBlockCount, matchedRootCount := model.FullTextSearchBlock(query, boxes, paths, types, method, orderBy, groupBy)
ret.Data = map[string]interface{}{
"blocks": blocks,
"matchedBlockCount": matchedBlockCount,

64
kernel/model/graph.go
View file

@ -18,6 +18,8 @@ package model
import (
"bytes"
"github.com/88250/lute/parse"
"github.com/siyuan-note/siyuan/kernel/util"
"math"
"strings"
"unicode/utf8"
@ -685,3 +687,65 @@ func nodeTitleLabel(node *GraphNode, blockContent string) {
node.Label = blockContent
}
}
func query2Stmt(queryStr string) (ret string) {
buf := bytes.Buffer{}
if util.IsIDPattern(queryStr) {
buf.WriteString("id = '" + queryStr + "'")
} else {
var tags []string
luteEngine := NewLute()
t := parse.Inline("", []byte(queryStr), luteEngine.ParseOptions)
ast.Walk(t.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
if !entering {
return ast.WalkContinue
}
if ast.NodeTag == n.Type || (n.IsTextMarkType("tag")) {
tags = append(tags, n.Text())
}
return ast.WalkContinue
})
for _, tag := range tags {
queryStr = strings.ReplaceAll(queryStr, "#"+tag+"#", "")
}
parts := strings.Split(queryStr, " ")
for i, part := range parts {
if "" == part {
continue
}
part = strings.ReplaceAll(part, "'", "''")
buf.WriteString("(content LIKE '%" + part + "%'")
buf.WriteString(Conf.Search.NAMFilter(part))
buf.WriteString(")")
if i < len(parts)-1 {
buf.WriteString(" AND ")
}
}
if 0 < len(tags) {
if 0 < buf.Len() {
buf.WriteString(" OR ")
}
for i, tag := range tags {
buf.WriteString("(content LIKE '%#" + tag + "#%')")
if i < len(tags)-1 {
buf.WriteString(" AND ")
}
}
buf.WriteString(" OR ")
for i, tag := range tags {
buf.WriteString("ial LIKE '%tags=\"%" + tag + "%\"%'")
if i < len(tags)-1 {
buf.WriteString(" AND ")
}
}
}
}
if 1 > buf.Len() {
buf.WriteString("1=1")
}
ret = buf.String()
return
}

235
kernel/model/search.go
View file

@ -312,31 +312,32 @@ func FindReplace(keyword, replacement string, ids []string, method int) (err err
return
}
func FullTextSearchBlock(query string, boxes, paths []string, types map[string]bool, method int, groupBy int) (ret []*Block, matchedBlockCount, matchedRootCount int) {
func FullTextSearchBlock(query string, boxes, paths []string, types map[string]bool, method, orderBy, groupBy int) (ret []*Block, matchedBlockCount, matchedRootCount int) {
// method：0：文本，1：查询语法，2：SQL，3：正则表达式
// orderBy: 0：按块类型（默认），1：按创建时间升序，2：按创建时间降序，3：按更新时间升序，4：按更新时间降序，5：按内容顺序（仅在按文档分组时）
// groupBy：0：不分组，1：按文档分组
query = strings.TrimSpace(query)
beforeLen := 36
var blocks []*Block
orderByClause := buildOrderBy(orderBy)
switch method {
case 1: // 查询语法
filter := buildTypeFilter(types)
boxFilter := buildBoxesFilter(boxes)
pathFilter := buildPathsFilter(paths)
blocks, matchedBlockCount, matchedRootCount = fullTextSearch(query, boxFilter, pathFilter, filter, beforeLen, true)
blocks, matchedBlockCount, matchedRootCount = fullTextSearchByQuerySyntax(query, boxFilter, pathFilter, filter, orderByClause, beforeLen)
case 2: // SQL
blocks, matchedBlockCount, matchedRootCount = searchBySQL(query, beforeLen)
case 3: // 正则表达式
typeFilter := buildTypeFilter(types)
boxFilter := buildBoxesFilter(boxes)
pathFilter := buildPathsFilter(paths)
blocks, matchedBlockCount, matchedRootCount = fullTextSearchByRegexp(query, boxFilter, pathFilter, typeFilter, beforeLen)
blocks, matchedBlockCount, matchedRootCount = fullTextSearchByRegexp(query, boxFilter, pathFilter, typeFilter, orderByClause, beforeLen)
default: // 文本
filter := buildTypeFilter(types)
boxFilter := buildBoxesFilter(boxes)
pathFilter := buildPathsFilter(paths)
blocks, matchedBlockCount, matchedRootCount = fullTextSearch(query, boxFilter, pathFilter, filter, beforeLen, false)
blocks, matchedBlockCount, matchedRootCount = fullTextSearchByKeyword(query, boxFilter, pathFilter, filter, orderByClause, beforeLen)
}
switch groupBy {
@ -421,6 +422,21 @@ func buildPathsFilter(paths []string) string {
return builder.String()
}
func buildOrderBy(orderBy int) string {
switch orderBy {
case 1:
return "ORDER BY created ASC"
case 2:
return "ORDER BY created DESC"
case 3:
return "ORDER BY updated ASC"
case 4:
return "ORDER BY updated DESC"
default:
return "ORDER BY sort ASC"
}
}
func buildTypeFilter(types map[string]bool) string {
s := conf.NewSearch()
if err := copier.Copy(s, Conf.Search); nil != err {
@ -523,6 +539,85 @@ func fullTextSearchRefBlock(keyword string, beforeLen int) (ret []*Block) {
return
}
func fullTextSearchByQuerySyntax(query, boxFilter, pathFilter, typeFilter, orderBy string, beforeLen int) (ret []*Block, matchedBlockCount, matchedRootCount int) {
query = gulu.Str.RemoveInvisible(query)
if util.IsIDPattern(query) {
ret, matchedBlockCount, matchedRootCount = searchBySQL("SELECT * FROM `blocks` WHERE `id` = '"+query+"'", beforeLen)
return
}
return fullTextSearchByFTS(query, boxFilter, pathFilter, typeFilter, orderBy, beforeLen)
}
func fullTextSearchByKeyword(query, boxFilter, pathFilter, typeFilter string, orderBy string, beforeLen int) (ret []*Block, matchedBlockCount, matchedRootCount int) {
query = gulu.Str.RemoveInvisible(query)
if util.IsIDPattern(query) {
ret, matchedBlockCount, matchedRootCount = searchBySQL("SELECT * FROM `blocks` WHERE `id` = '"+query+"'", beforeLen)
return
}
query = stringQuery(query)
return fullTextSearchByFTS(query, boxFilter, pathFilter, typeFilter, orderBy, beforeLen)
}
func fullTextSearchByRegexp(exp, boxFilter, pathFilter, typeFilter, orderBy string, beforeLen int) (ret []*Block, matchedBlockCount, matchedRootCount int) {
exp = gulu.Str.RemoveInvisible(exp)
exp = regexp.QuoteMeta(exp)
fieldFilter := fieldRegexp(exp)
stmt := "SELECT * FROM `blocks` WHERE (" + fieldFilter + ") AND type IN " + typeFilter
stmt += boxFilter + pathFilter
stmt += " " + orderBy
stmt += " LIMIT " + strconv.Itoa(Conf.Search.Limit)
blocks := sql.SelectBlocksRawStmt(stmt, Conf.Search.Limit)
ret = fromSQLBlocks(&blocks, "", beforeLen)
if 1 > len(ret) {
ret = []*Block{}
}
matchedBlockCount, matchedRootCount = fullTextSearchCountByRegexp(exp, boxFilter, pathFilter, typeFilter)
return
}
func fullTextSearchCountByRegexp(exp, boxFilter, pathFilter, typeFilter string) (matchedBlockCount, matchedRootCount int) {
fieldFilter := fieldRegexp(exp)
stmt := "SELECT COUNT(id) AS `matches`, COUNT(DISTINCT(root_id)) AS `docs` FROM `blocks` WHERE " + fieldFilter + " AND type IN " + typeFilter
stmt += boxFilter + pathFilter
stmt += " LIMIT " + strconv.Itoa(Conf.Search.Limit)
result, _ := sql.Query(stmt)
if 1 > len(result) {
return
}
matchedBlockCount = int(result[0]["matches"].(int64))
matchedRootCount = int(result[0]["docs"].(int64))
return
}
func fullTextSearchByFTS(query, boxFilter, pathFilter, typeFilter, orderBy string, beforeLen int) (ret []*Block, matchedBlockCount, matchedRootCount int) {
table := "blocks_fts" // 大小写敏感
if !Conf.Search.CaseSensitive {
table = "blocks_fts_case_insensitive"
}
projections := "id, parent_id, root_id, hash, box, path, " +
"highlight(" + table + ", 6, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "') AS hpath, " +
"highlight(" + table + ", 7, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "') AS name, " +
"highlight(" + table + ", 8, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "') AS alias, " +
"highlight(" + table + ", 9, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "') AS memo, " +
"tag, " +
"highlight(" + table + ", 11, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "') AS content, " +
"fcontent, markdown, length, type, subtype, ial, sort, created, updated"
stmt := "SELECT " + projections + " FROM " + table + " WHERE " + table + " MATCH '" + columnFilter() + ":(" + query + ")' AND type IN " + typeFilter
stmt += boxFilter + pathFilter
stmt += " " + orderBy
stmt += " LIMIT " + strconv.Itoa(Conf.Search.Limit)
blocks := sql.SelectBlocksRawStmt(stmt, Conf.Search.Limit)
ret = fromSQLBlocks(&blocks, "", beforeLen)
if 1 > len(ret) {
ret = []*Block{}
}
matchedBlockCount, matchedRootCount = fullTextSearchCount(query, boxFilter, pathFilter, typeFilter)
return
}
func fullTextSearchCount(query, boxFilter, pathFilter, typeFilter string) (matchedBlockCount, matchedRootCount int) {
query = gulu.Str.RemoveInvisible(query)
if util.IsIDPattern(query) {
@ -551,136 +646,6 @@ func fullTextSearchCount(query, boxFilter, pathFilter, typeFilter string) (match
return
}
func fullTextSearch(query, boxFilter, pathFilter, typeFilter string, beforeLen int, querySyntax bool) (ret []*Block, matchedBlockCount, matchedRootCount int) {
query = gulu.Str.RemoveInvisible(query)
if util.IsIDPattern(query) {
ret, matchedBlockCount, matchedRootCount = searchBySQL("SELECT * FROM `blocks` WHERE `id` = '"+query+"'", beforeLen)
return
}
if !querySyntax {
query = stringQuery(query)
}
table := "blocks_fts" // 大小写敏感
if !Conf.Search.CaseSensitive {
table = "blocks_fts_case_insensitive"
}
projections := "id, parent_id, root_id, hash, box, path, " +
"highlight(" + table + ", 6, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "') AS hpath, " +
"highlight(" + table + ", 7, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "') AS name, " +
"highlight(" + table + ", 8, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "') AS alias, " +
"highlight(" + table + ", 9, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "') AS memo, " +
"tag, " +
"highlight(" + table + ", 11, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "') AS content, " +
"fcontent, markdown, length, type, subtype, ial, sort, created, updated"
stmt := "SELECT " + projections + " FROM " + table + " WHERE " + table + " MATCH '" + columnFilter() + ":(" + query + ")' AND type IN " + typeFilter
stmt += boxFilter + pathFilter
stmt += " ORDER BY sort ASC, rank ASC LIMIT " + strconv.Itoa(Conf.Search.Limit)
blocks := sql.SelectBlocksRawStmt(stmt, Conf.Search.Limit)
ret = fromSQLBlocks(&blocks, "", beforeLen)
if 1 > len(ret) {
ret = []*Block{}
}
matchedBlockCount, matchedRootCount = fullTextSearchCount(query, boxFilter, pathFilter, typeFilter)
return
}
func fullTextSearchByRegexp(exp, boxFilter, pathFilter, typeFilter string, beforeLen int) (ret []*Block, matchedBlockCount, matchedRootCount int) {
exp = gulu.Str.RemoveInvisible(exp)
exp = regexp.QuoteMeta(exp)
fieldFilter := fieldRegexp(exp)
stmt := "SELECT * FROM `blocks` WHERE (" + fieldFilter + ") AND type IN " + typeFilter
stmt += boxFilter + pathFilter
stmt += " ORDER BY sort ASC LIMIT " + strconv.Itoa(Conf.Search.Limit)
blocks := sql.SelectBlocksRawStmt(stmt, Conf.Search.Limit)
ret = fromSQLBlocks(&blocks, "", beforeLen)
if 1 > len(ret) {
ret = []*Block{}
}
matchedBlockCount, matchedRootCount = fullTextSearchCountByRegexp(exp, boxFilter, pathFilter, typeFilter)
return
}
func fullTextSearchCountByRegexp(exp, boxFilter, pathFilter, typeFilter string) (matchedBlockCount, matchedRootCount int) {
fieldFilter := fieldRegexp(exp)
stmt := "SELECT COUNT(id) AS `matches`, COUNT(DISTINCT(root_id)) AS `docs` FROM `blocks` WHERE " + fieldFilter + " AND type IN " + typeFilter
stmt += boxFilter + pathFilter
stmt += " LIMIT " + strconv.Itoa(Conf.Search.Limit)
result, _ := sql.Query(stmt)
if 1 > len(result) {
return
}
matchedBlockCount = int(result[0]["matches"].(int64))
matchedRootCount = int(result[0]["docs"].(int64))
return
}
func query2Stmt(queryStr string) (ret string) {
buf := bytes.Buffer{}
if util.IsIDPattern(queryStr) {
buf.WriteString("id = '" + queryStr + "'")
} else {
var tags []string
luteEngine := NewLute()
t := parse.Inline("", []byte(queryStr), luteEngine.ParseOptions)
ast.Walk(t.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
if !entering {
return ast.WalkContinue
}
if ast.NodeTag == n.Type || (n.IsTextMarkType("tag")) {
tags = append(tags, n.Text())
}
return ast.WalkContinue
})
for _, tag := range tags {
queryStr = strings.ReplaceAll(queryStr, "#"+tag+"#", "")
}
parts := strings.Split(queryStr, " ")
for i, part := range parts {
if "" == part {
continue
}
part = strings.ReplaceAll(part, "'", "''")
buf.WriteString("(content LIKE '%" + part + "%'")
buf.WriteString(Conf.Search.NAMFilter(part))
buf.WriteString(")")
if i < len(parts)-1 {
buf.WriteString(" AND ")
}
}
if 0 < len(tags) {
if 0 < buf.Len() {
buf.WriteString(" OR ")
}
for i, tag := range tags {
buf.WriteString("(content LIKE '%#" + tag + "#%')")
if i < len(tags)-1 {
buf.WriteString(" AND ")
}
}
buf.WriteString(" OR ")
for i, tag := range tags {
buf.WriteString("ial LIKE '%tags=\"%" + tag + "%\"%'")
if i < len(tags)-1 {
buf.WriteString(" AND ")
}
}
}
}
if 1 > buf.Len() {
buf.WriteString("1=1")
}
ret = buf.String()
return
}
func markSearch(text string, keyword string, beforeLen int) (marked string, score float64) {
if 0 == len(keyword) {
marked = text