0ct0pu5/siyuan
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

♻️ Supports converting network assets in the database to local https://github.com/siyuan-note/siyuan/issues/12096

Browse source
This commit is contained in:
Daniel 2024-07-26 18:41:46 +08:00
parent 2a7702a78b
commit df8d70995c
No known key found for this signature in database
GPG key ID: 86211BA83DF03017
2 changed files with 78 additions and 231 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
kernel/api/format.go
View file

@ -35,7 +35,7 @@ func netAssets2LocalAssets(c *gin.Context) {
}
id := arg["id"].(string)
err := model.NetAssets2LocalAssets(id)
err := model.NetAssets2LocalAssets(id, false, "")
if nil != err {
ret.Code = -1
ret.Msg = err.Error()
@ -58,7 +58,7 @@ func netImg2LocalAssets(c *gin.Context) {
if urlArg := arg["url"]; nil != urlArg {
url = urlArg.(string)
}
err := model.NetImg2LocalAssets(id, url)
err := model.NetAssets2LocalAssets(id, true, url)
if nil != err {
ret.Code = -1
ret.Msg = err.Error()

305
kernel/model/assets.go
View file

@ -75,182 +75,7 @@ func DocImageAssets(rootID string) (ret []string, err error) {
return
}
func NetImg2LocalAssets(rootID, originalURL string) (err error) {
tree, err := LoadTreeByBlockID(rootID)
if nil != err {
return
}
var files int
msgId := gulu.Rand.String(7)
docDirLocalPath := filepath.Join(util.DataDir, tree.Box, path.Dir(tree.Path))
assetsDirPath := getAssetsDir(filepath.Join(util.DataDir, tree.Box), docDirLocalPath)
if !gulu.File.IsExist(assetsDirPath) {
if err = os.MkdirAll(assetsDirPath, 0755); nil != err {
return
}
}
browserClient := req.C().
SetUserAgent(util.UserAgent).
SetTimeout(30 * time.Second).
EnableInsecureSkipVerify(). // HTTPS certificate is no longer verified when `Convert network images to local images` https://github.com/siyuan-note/siyuan/issues/9080
SetProxy(httpclient.ProxyFromEnvironment)
ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
if !entering {
return ast.WalkContinue
}
if ast.NodeImage == n.Type {
linkDest := n.ChildByType(ast.NodeLinkDest)
linkText := n.ChildByType(ast.NodeLinkText)
if nil == linkText {
linkText = &ast.Node{Type: ast.NodeLinkText, Tokens: []byte("image")}
if openBracket := n.ChildByType(ast.NodeOpenBracket); nil != openBracket {
openBracket.InsertAfter(linkText)
}
}
dest := linkDest.Tokens
if util.IsAssetLinkDest(dest) {
return ast.WalkSkipChildren
}
if bytes.HasPrefix(bytes.ToLower(dest), []byte("file://")) {
// `网络图片转换为本地图片` 支持处理 `file://` 本地路径图片 https://github.com/siyuan-note/siyuan/issues/6546
u := string(dest)[7:]
unescaped, _ := url.PathUnescape(u)
if unescaped != u {
// `Convert network images/assets to local` supports URL-encoded local file names https://github.com/siyuan-note/siyuan/issues/9929
u = unescaped
}
if strings.Contains(u, ":") {
u = strings.TrimPrefix(u, "/")
}
if !gulu.File.IsExist(u) || gulu.File.IsDir(u) {
return ast.WalkSkipChildren
}
name := filepath.Base(u)
name = util.FilterUploadFileName(name)
name = util.TruncateLenFileName(name)
if 1 > len(bytes.TrimSpace(linkText.Tokens)) {
linkText.Tokens = []byte(name)
}
name = "net-img-" + name
name = util.AssetName(name)
writePath := filepath.Join(assetsDirPath, name)
if err = filelock.Copy(u, writePath); nil != err {
logging.LogErrorf("copy [%s] to [%s] failed: %s", u, writePath, err)
return ast.WalkSkipChildren
}
linkDest.Tokens = []byte("assets/" + name)
files++
return ast.WalkSkipChildren
}
if bytes.HasPrefix(bytes.ToLower(dest), []byte("https://")) || bytes.HasPrefix(bytes.ToLower(dest), []byte("http://")) || bytes.HasPrefix(dest, []byte("//")) {
if bytes.HasPrefix(dest, []byte("//")) {
// `Convert network images to local` supports `//` https://github.com/siyuan-note/siyuan/issues/10598
dest = append([]byte("https:"), dest...)
}
u := string(dest)
if strings.Contains(u, "qpic.cn") {
// 改进 `网络图片转换为本地图片` 微信图片拉取 https://github.com/siyuan-note/siyuan/issues/5052
if strings.Contains(u, "http://") {
u = strings.Replace(u, "http://", "https://", 1)
}
// 改进 `网络图片转换为本地图片` 微信图片拉取 https://github.com/siyuan-note/siyuan/issues/6431
// 下面这部分需要注释掉，否则会导致响应 400
//if strings.HasSuffix(u, "/0") {
// u = strings.Replace(u, "/0", "/640", 1)
//} else if strings.Contains(u, "/0?") {
// u = strings.Replace(u, "/0?", "/640?", 1)
//}
}
util.PushUpdateMsg(msgId, fmt.Sprintf(Conf.Language(119), u), 15000)
request := browserClient.R()
request.SetRetryCount(1).SetRetryFixedInterval(3 * time.Second)
if "" != originalURL {
request.SetHeader("Referer", originalURL) // 改进浏览器剪藏扩展转换本地图片成功率 https://github.com/siyuan-note/siyuan/issues/7464
}
resp, reqErr := request.Get(u)
if nil != reqErr {
logging.LogErrorf("download net img [%s] failed: %s", u, reqErr)
return ast.WalkSkipChildren
}
if 200 != resp.StatusCode {
logging.LogErrorf("download net img [%s] failed: %d", u, resp.StatusCode)
return ast.WalkSkipChildren
}
data, repErr := resp.ToBytes()
if nil != repErr {
logging.LogErrorf("download net img [%s] failed: %s", u, repErr)
return ast.WalkSkipChildren
}
var name string
if strings.Contains(u, "?") {
name = u[:strings.Index(u, "?")]
name = path.Base(name)
} else {
name = path.Base(u)
}
if strings.Contains(name, "#") {
name = name[:strings.Index(name, "#")]
}
name, _ = url.PathUnescape(name)
ext := path.Ext(name)
if "" == ext {
if mtype := mimetype.Detect(data); nil != mtype {
ext = mtype.Extension()
}
}
if "" == ext {
contentType := resp.Header.Get("Content-Type")
exts, _ := mime.ExtensionsByType(contentType)
if 0 < len(exts) {
ext = exts[0]
}
}
name = strings.TrimSuffix(name, ext)
name = util.FilterUploadFileName(name)
name = util.TruncateLenFileName(name)
if 1 > len(bytes.TrimSpace(linkText.Tokens)) {
linkText.Tokens = []byte(name)
}
name = "net-img-" + name + "-" + ast.NewNodeID() + ext
writePath := filepath.Join(assetsDirPath, name)
if err = filelock.WriteFile(writePath, data); nil != err {
logging.LogErrorf("write downloaded net img [%s] to local assets [%s] failed: %s", u, writePath, err)
return ast.WalkSkipChildren
}
linkDest.Tokens = []byte("assets/" + name)
files++
}
return ast.WalkSkipChildren
}
return ast.WalkContinue
})
if 0 < files {
util.PushUpdateMsg(msgId, Conf.Language(113), 7000)
if err = writeTreeUpsertQueue(tree); nil != err {
return
}
util.PushUpdateMsg(msgId, fmt.Sprintf(Conf.Language(120), files), 5000)
} else {
util.PushUpdateMsg(msgId, Conf.Language(121), 3000)
}
return
}
func NetAssets2LocalAssets(rootID string) (err error) {
func NetAssets2LocalAssets(rootID string, onlyImg bool, originalURL string) (err error) {
tree, err := LoadTreeByBlockID(rootID)
if nil != err {
return
@ -273,32 +98,15 @@ func NetAssets2LocalAssets(rootID string) (err error) {
EnableInsecureSkipVerify().
SetProxy(httpclient.ProxyFromEnvironment)
ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
if !entering || (ast.NodeLinkDest != n.Type && !n.IsTextMarkType("a") && ast.NodeAudio != n.Type && ast.NodeVideo != n.Type) {
return ast.WalkContinue
destNodes := getRemoteAssetsLinkDestsInTree(tree, onlyImg)
for _, destNode := range destNodes {
dest := getRemoteAssetsLinkDest(destNode, onlyImg)
if "" == dest {
continue
}
var dest []byte
if ast.NodeLinkDest == n.Type {
dest = n.Tokens
} else if n.IsTextMarkType("a") {
dest = []byte(n.TextMarkAHref)
} else if ast.NodeAudio == n.Type || ast.NodeVideo == n.Type {
if srcIndex := bytes.Index(n.Tokens, []byte("src=\"")); 0 < srcIndex {
src := n.Tokens[srcIndex+len("src=\""):]
if srcIndex = bytes.Index(src, []byte("\"")); 0 < srcIndex {
src = src[:bytes.Index(src, []byte("\""))]
dest = bytes.TrimSpace(src)
}
}
}
if util.IsAssetLinkDest(dest) {
return ast.WalkContinue
}
if bytes.HasPrefix(bytes.ToLower(dest), []byte("file://")) { // 处理本地文件链接
u := string(dest)[7:]
if strings.HasPrefix(strings.ToLower(dest), "file://") { // 处理本地文件链接
u := dest[7:]
unescaped, _ := url.PathUnescape(u)
if unescaped != u {
// `Convert network images/assets to local` supports URL-encoded local file names https://github.com/siyuan-note/siyuan/issues/9929
@ -309,7 +117,7 @@ func NetAssets2LocalAssets(rootID string) (err error) {
}
if !gulu.File.IsExist(u) || gulu.File.IsDir(u) {
return ast.WalkContinue
continue
}
name := filepath.Base(u)
@ -320,27 +128,21 @@ func NetAssets2LocalAssets(rootID string) (err error) {
writePath := filepath.Join(assetsDirPath, name)
if err = filelock.Copy(u, writePath); nil != err {
logging.LogErrorf("copy [%s] to [%s] failed: %s", u, writePath, err)
return ast.WalkContinue
continue
}
if ast.NodeLinkDest == n.Type {
n.Tokens = []byte("assets/" + name)
} else if n.IsTextMarkType("a") {
n.TextMarkAHref = "assets/" + name
} else if ast.NodeAudio == n.Type || ast.NodeVideo == n.Type {
n.Tokens = bytes.ReplaceAll(n.Tokens, dest, []byte("assets/"+name))
}
setAssetsLinkDest(destNode, dest, "assets/"+name)
files++
return ast.WalkContinue
continue
}
if bytes.HasPrefix(bytes.ToLower(dest), []byte("https://")) || bytes.HasPrefix(bytes.ToLower(dest), []byte("http://")) || bytes.HasPrefix(dest, []byte("//")) {
if bytes.HasPrefix(dest, []byte("//")) {
if strings.HasPrefix(strings.ToLower(dest), "https://") || strings.HasPrefix(strings.ToLower(dest), "http://") || strings.HasPrefix(dest, "//") {
if strings.HasPrefix(dest, "//") {
// `Convert network images to local` supports `//` https://github.com/siyuan-note/siyuan/issues/10598
dest = append([]byte("https:"), dest...)
dest = "https:" + dest
}
u := string(dest)
u := dest
if strings.Contains(u, "qpic.cn") {
// 改进 `网络图片转换为本地图片` 微信图片拉取 https://github.com/siyuan-note/siyuan/issues/5052
if strings.Contains(u, "http://") {
@ -358,30 +160,33 @@ func NetAssets2LocalAssets(rootID string) (err error) {
util.PushUpdateMsg(msgId, fmt.Sprintf(Conf.Language(119), u), 15000)
request := browserClient.R()
request.SetRetryCount(1).SetRetryFixedInterval(3 * time.Second)
if "" != originalURL {
request.SetHeader("Referer", originalURL) // 改进浏览器剪藏扩展转换本地图片成功率 https://github.com/siyuan-note/siyuan/issues/7464
}
resp, reqErr := request.Get(u)
if strings.Contains(strings.ToLower(resp.GetContentType()), "text/html") {
// 忽略超链接网页 `Convert network assets to local` no longer process webpage https://github.com/siyuan-note/siyuan/issues/9965
return ast.WalkContinue
continue
}
if nil != reqErr {
logging.LogErrorf("download network asset [%s] failed: %s", u, reqErr)
return ast.WalkContinue
continue
}
if 200 != resp.StatusCode {
logging.LogErrorf("download network asset [%s] failed: %d", u, resp.StatusCode)
return ast.WalkContinue
continue
}
if 1024*1024*96 < resp.ContentLength {
logging.LogWarnf("network asset [%s]' size [%s] is large then [96 MB], ignore it", u, humanize.IBytes(uint64(resp.ContentLength)))
return ast.WalkContinue
continue
}
data, repErr := resp.ToBytes()
if nil != repErr {
logging.LogErrorf("download network asset [%s] failed: %s", u, repErr)
return ast.WalkContinue
continue
}
var name string
if strings.Contains(u, "?") {
@ -414,20 +219,14 @@ func NetAssets2LocalAssets(rootID string) (err error) {
writePath := filepath.Join(assetsDirPath, name)
if err = filelock.WriteFile(writePath, data); nil != err {
logging.LogErrorf("write downloaded network asset [%s] to local asset [%s] failed: %s", u, writePath, err)
return ast.WalkContinue
continue
}
if ast.NodeLinkDest == n.Type {
n.Tokens = []byte("assets/" + name)
} else if n.IsTextMarkType("a") {
n.TextMarkAHref = "assets/" + name
} else if ast.NodeAudio == n.Type || ast.NodeVideo == n.Type {
n.Tokens = bytes.ReplaceAll(n.Tokens, dest, []byte("assets/"+name))
}
setAssetsLinkDest(destNode, dest, "assets/"+name)
files++
continue
}
return ast.WalkContinue
})
}
if 0 < files {
util.PushUpdateMsg(msgId, Conf.Language(113), 7000)
@ -1242,6 +1041,54 @@ func assetsLinkDestsInNode(node *ast.Node) (ret []string) {
return
}
func setAssetsLinkDest(node *ast.Node, oldDest, dest string) {
if ast.NodeLinkDest == node.Type {
node.Tokens = bytes.ReplaceAll(node.Tokens, []byte(oldDest), []byte(dest))
} else if node.IsTextMarkType("a") {
node.TextMarkAHref = strings.ReplaceAll(node.TextMarkAHref, oldDest, dest)
} else if ast.NodeAudio == node.Type || ast.NodeVideo == node.Type {
node.Tokens = bytes.ReplaceAll(node.Tokens, []byte(oldDest), []byte(dest))
}
}
func getRemoteAssetsLinkDest(node *ast.Node, onlyImg bool) (ret string) {
if onlyImg {
if ast.NodeLinkDest == node.Type && node.ParentIs(ast.NodeImage) {
ret = string(node.Tokens)
}
} else {
if ast.NodeLinkDest == node.Type {
ret = string(node.Tokens)
} else if node.IsTextMarkType("a") {
ret = node.TextMarkAHref
} else if ast.NodeAudio == node.Type || ast.NodeVideo == node.Type {
ret = treenode.GetNodeSrcTokens(node)
}
}
if util.IsAssetLinkDest([]byte(ret)) {
ret = ""
}
return
}
func getRemoteAssetsLinkDestsInTree(tree *parse.Tree, onlyImg bool) (nodes []*ast.Node) {
ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
if !entering {
return ast.WalkContinue
}
dest := getRemoteAssetsLinkDest(n, onlyImg)
if "" == dest {
return ast.WalkContinue
}
nodes = append(nodes, n)
return ast.WalkContinue
})
return
}
// allAssetAbsPaths 返回 asset 相对路径（assets/xxx）到绝对路径（F:\SiYuan\data\assets\xxx）的映射。
func allAssetAbsPaths() (assetsAbsPathMap map[string]string, err error) {
notebooks, err := ListNotebooks()