:recycle: 一些字符串操作函数移到 Gulu 项目里

kernel/go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/88250/clipboard v0.1.5
 	github.com/88250/css v0.1.2
 	github.com/88250/flock v0.8.2
-	github.com/88250/gulu v1.2.1
+	github.com/88250/gulu v1.2.2
 	github.com/88250/lute v1.7.4-0.20220525011519-3148f42c174b
 	github.com/88250/melody v0.0.0-20201115062536-c0b3394adcd1
 	github.com/88250/pdfcpu v0.3.13

kernel/go.sum

@@ -48,8 +48,8 @@ github.com/88250/flock v0.8.2/go.mod h1:k+PZxETAUe4vLZx3R39ykvQCIlwHhc7AI2P2NUQV
 github.com/88250/go-sqlite3 v1.14.13-0.20220412041952-88c3aaa8595e h1:uXi4QLKI/mswcXuzD+wBjJMkj1C3hK5Tgl3hF6MJpbo=
 github.com/88250/go-sqlite3 v1.14.13-0.20220412041952-88c3aaa8595e/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
 github.com/88250/gulu v1.2.0/go.mod h1:ZhEJ98UjR2y7j2toGj4/b+1rRELcZFQAPq/Yjyin2yY=
-github.com/88250/gulu v1.2.1 h1:omRzVo2IToCL1sclmAy3Xq81vAW7nY/vZN1FopAvJtg=
-github.com/88250/gulu v1.2.1/go.mod h1:I1qBzsksFL2ciGSuqDE7R3XW4BUMrfDgOvSXEk7FsAI=
+github.com/88250/gulu v1.2.2 h1:LsuDSXI4GjqAD2XasG/DJEHuboKXU2lhZ9w0mnfJj3Y=
+github.com/88250/gulu v1.2.2/go.mod h1:I1qBzsksFL2ciGSuqDE7R3XW4BUMrfDgOvSXEk7FsAI=
 github.com/88250/lute v1.7.4-0.20220426011157-34c9bfa2e148/go.mod h1:Bdu9LRNjQhtL3TftbtpjIWTwDVAXoS7AD8QsZQPk7zo=
 github.com/88250/lute v1.7.4-0.20220525011519-3148f42c174b h1:UwGrvVWNkXpyv8FkazWeV0brdmhM7FpWE3qMfqF1K+4=
 github.com/88250/lute v1.7.4-0.20220525011519-3148f42c174b/go.mod h1:Bdu9LRNjQhtL3TftbtpjIWTwDVAXoS7AD8QsZQPk7zo=

kernel/model/css.go

@@ -236,7 +236,7 @@ func WriteCustomCSS(themeName string, cssMap map[string]interface{}) (err error)
 		return
 	}
 
-	cssData := util.RemoveInvisible(string(data))
+	cssData := gulu.Str.RemoveInvisible(string(data))
 	customStyleSheet := css.Parse(cssData)
 
 	buf := &bytes.Buffer{}

kernel/model/file.go

@@ -540,7 +540,7 @@ func GetDoc(id string, index int, keyword string, mode int, size int) (blockCoun
 			var tmp []string
 			for _, k := range virtualBlockRefKeywords {
 				repl := "<span data-type=\"virtual-block-ref\">" + k + "</span>"
-				if util.IsASCII(k) {
+				if gulu.Str.IsASCII(k) {
 					tmp = append(tmp, " "+k, " "+repl)
 					tmp = append(tmp, " "+k+" ", " "+repl+" ")
 					tmp = append(tmp, k+" ", repl+" ")
@@ -617,7 +617,7 @@ func GetDoc(id string, index int, keyword string, mode int, size int) (blockCoun
 								blockKeys = append(blockKeys, alias)
 							}
 							if 0 < len(blockKeys) {
-								keys := util.SubstringsBetween(newContent, "<span data-type=\"virtual-block-ref\">", "</span>")
+								keys := gulu.Str.SubstringsBetween(newContent, "<span data-type=\"virtual-block-ref\">", "</span>")
 								for _, k := range keys {
 									if gulu.Str.Contains(k, blockKeys) {
 										return ast.WalkContinue
@@ -1173,7 +1173,7 @@ func RenameDoc(boxID, p, title string) (err error) {
 		return
 	}
 
-	title = util.RemoveInvisible(title)
+	title = gulu.Str.RemoveInvisible(title)
 	oldTitle := tree.Root.IALAttr("title")
 	if oldTitle == title {
 		return
@@ -1287,7 +1287,7 @@ func CreateDailyNote(boxID string) (p string, err error) {
 }
 
 func createDoc(boxID, p, title, dom string) (err error) {
-	title = util.RemoveInvisible(title)
+	title = gulu.Str.RemoveInvisible(title)
 
 	baseName := strings.TrimSpace(path.Base(p))
 	if "" == strings.TrimSuffix(baseName, ".sy") {

kernel/model/search.go

@@ -224,7 +224,7 @@ func searchFilter(types map[string]bool) string {
 }
 
 func searchBySQL(stmt string, beforeLen int) (ret []*Block) {
-	stmt = util.RemoveInvisible(stmt)
+	stmt = gulu.Str.RemoveInvisible(stmt)
 	blocks := sql.SelectBlocksRawStmt(stmt, Conf.Search.Limit)
 	ret = fromSQLBlocks(&blocks, "", beforeLen)
 	if 1 > len(ret) {
@@ -234,7 +234,7 @@ func searchBySQL(stmt string, beforeLen int) (ret []*Block) {
 }
 
 func fullTextSearchRefBlock(keyword string, beforeLen int) (ret []*Block) {
-	keyword = util.RemoveInvisible(keyword)
+	keyword = gulu.Str.RemoveInvisible(keyword)
 
 	if util.IsIDPattern(keyword) {
 		ret = searchBySQL("SELECT * FROM `blocks` WHERE `id` = '"+keyword+"'", 36)
@@ -282,7 +282,7 @@ func fullTextSearchRefBlock(keyword string, beforeLen int) (ret []*Block) {
 }
 
 func fullTextSearch(query, box, path, filter string, beforeLen int, querySyntax bool) (ret []*Block) {
-	query = util.RemoveInvisible(query)
+	query = gulu.Str.RemoveInvisible(query)
 	if util.IsIDPattern(query) {
 		ret = searchBySQL("SELECT * FROM `blocks` WHERE `id` = '"+query+"'", beforeLen)
 		return
@@ -394,7 +394,7 @@ func markSearch(text string, keyword string, beforeLen int) (marked string, scor
 			return
 		}
 
-		keywords := util.SubstringsBetween(marked, "__@mark__", "__mark@__")
+		keywords := gulu.Str.SubstringsBetween(marked, "__@mark__", "__mark@__")
 		keywords = util.RemoveDuplicatedElem(keywords)
 		keyword = strings.Join(keywords, search.TermSep)
 		marked = strings.ReplaceAll(marked, "__@mark__", "")

kernel/model/sync.go

@@ -1137,7 +1137,7 @@ func CreateCloudSyncDir(name string) (err error) {
 	defer syncLock.Unlock()
 
 	name = strings.TrimSpace(name)
-	name = util.RemoveInvisible(name)
+	name = gulu.Str.RemoveInvisible(name)
 	if !IsValidCloudDirName(name) {
 		return errors.New(Conf.Language(37))
 	}

kernel/treenode/marker.go

@@ -17,12 +17,12 @@
 package treenode
 
 import (
+	"github.com/88250/gulu"
 	"github.com/88250/lute/lex"
-	"github.com/siyuan-note/siyuan/kernel/util"
 )
 
 func ContainsMarker(str string) bool {
-	if !util.IsASCII(str) {
+	if !gulu.Str.IsASCII(str) {
 		return false
 	}
 

kernel/util/string.go

@@ -15,99 +15,3 @@
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 package util
-
-import (
-	"strings"
-	"unicode"
-)
-
-func RemoveInvisible(str string) string {
-	str = strings.ReplaceAll(str, "\u00A0", " ") // NBSP 转换为普通空格
-	str = RemoveZeroWidthCharacters(str)
-	str = stripCtlFromUTF8(str)
-	return str
-}
-
-func stripCtlFromUTF8(str string) string {
-	return strings.Map(func(r rune) rune {
-		if r >= 32 && r != 127 {
-			return r
-		}
-		return -1
-	}, str)
-}
-
-const (
-	// ZWSP represents zero-width space.
-	ZWSP = '\u200B'
-
-	// ZWNBSP represents zero-width no-break space.
-	ZWNBSP = '\uFEFF'
-
-	// ZWJ represents zero-width joiner.
-	ZWJ = '\u200D'
-
-	// ZWNJ represents zero-width non-joiner.
-	ZWNJ = '\u200C'
-
-	empty = ""
-)
-
-var replacer = strings.NewReplacer(string(ZWSP), empty,
-	string(ZWNBSP), empty,
-	string(ZWJ), empty,
-	string(ZWNJ), empty)
-
-// HasZeroWidthCharacters reports whether string s contains zero-width characters.
-func HasZeroWidthCharacters(s string) bool {
-	return strings.ContainsRune(s, ZWSP) ||
-		strings.ContainsRune(s, ZWNBSP) ||
-		strings.ContainsRune(s, ZWJ) ||
-		strings.ContainsRune(s, ZWNJ)
-}
-
-// RemoveZeroWidthCharacters removes all zero-width characters from string s.
-func RemoveZeroWidthCharacters(s string) string {
-	return replacer.Replace(s)
-}
-
-// RemoveZeroWidthSpace removes zero-width space characters from string s.
-func RemoveZeroWidthSpace(s string) string {
-	return strings.Replace(s, string(ZWSP), empty, -1)
-}
-
-// RemoveZeroWidthNoBreakSpace removes zero-width no-break space characters from string s.
-func RemoveZeroWidthNoBreakSpace(s string) string {
-	return strings.Replace(s, string(ZWNBSP), empty, -1)
-}
-
-// RemoveZeroWidthJoiner removes zero-width joiner characters from string s.
-func RemoveZeroWidthJoiner(s string) string {
-	return strings.Replace(s, string(ZWJ), empty, -1)
-}
-
-// RemoveZeroWidthNonJoiner removes zero-width non-joiner characters from string s.
-func RemoveZeroWidthNonJoiner(s string) string {
-	return strings.Replace(s, string(ZWNJ), empty, -1)
-}
-
-func IsASCII(s string) bool {
-	for i := 0; i < len(s); i++ {
-		if s[i] > unicode.MaxASCII {
-			return false
-		}
-	}
-	return true
-}
-
-func SubstringsBetween(str, start, end string) (ret []string) {
-	parts := strings.Split(str, start)
-	for _, p := range parts {
-		if !strings.Contains(p, end) {
-			continue
-		}
-		parts2 := strings.Split(p, end)
-		ret = append(ret, parts2[0])
-	}
-	return
-}