// SiYuan - Refactor your thinking // Copyright (c) 2020-present, b3log.org // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Affero General Public License for more details. // // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . package bazaar import ( "bytes" "errors" "fmt" "os" "path/filepath" "strings" "sync" "time" "github.com/88250/gulu" "github.com/88250/lute" "github.com/araddon/dateparse" "github.com/imroc/req/v3" gcache "github.com/patrickmn/go-cache" "github.com/siyuan-note/filelock" "github.com/siyuan-note/httpclient" "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/util" "golang.org/x/mod/semver" textUnicode "golang.org/x/text/encoding/unicode" "golang.org/x/text/transform" ) type DisplayName struct { Default string `json:"default"` ZhCN string `json:"zh_CN"` EnUS string `json:"en_US"` ZhCHT string `json:"zh_CHT"` } type Description struct { Default string `json:"default"` ZhCN string `json:"zh_CN"` EnUS string `json:"en_US"` ZhCHT string `json:"zh_CHT"` } type Readme struct { Default string `json:"default"` ZhCN string `json:"zh_CN"` EnUS string `json:"en_US"` ZhCHT string `json:"zh_CHT"` } type Funding struct { OpenCollective string `json:"openCollective"` Patreon string `json:"patreon"` GitHub string `json:"github"` Custom []string `json:"custom"` } type Package struct { Author string `json:"author"` URL string `json:"url"` Version string `json:"version"` MinAppVersion string `json:"minAppVersion"` Backends []string `json:"backends"` Frontends []string `json:"frontends"` DisplayName *DisplayName `json:"displayName"` Description *Description `json:"description"` Readme *Readme `json:"readme"` Funding *Funding `json:"funding"` Keywords []string `json:"keywords"` PreferredFunding string `json:"preferredFunding"` PreferredName string `json:"preferredName"` PreferredDesc string `json:"preferredDesc"` PreferredReadme string `json:"preferredReadme"` Name string `json:"name"` RepoURL string `json:"repoURL"` RepoHash string `json:"repoHash"` PreviewURL string `json:"previewURL"` PreviewURLThumb string `json:"previewURLThumb"` IconURL string `json:"iconURL"` Installed bool `json:"installed"` Outdated bool `json:"outdated"` Current bool `json:"current"` Updated string `json:"updated"` Stars int `json:"stars"` OpenIssues int `json:"openIssues"` Size int64 `json:"size"` HSize string `json:"hSize"` InstallSize int64 `json:"installSize"` HInstallSize string `json:"hInstallSize"` HInstallDate string `json:"hInstallDate"` HUpdated string `json:"hUpdated"` Downloads int `json:"downloads"` Incompatible bool `json:"incompatible"` } type StagePackage struct { Author string `json:"author"` URL string `json:"url"` Version string `json:"version"` Description *Description `json:"description"` Readme *Readme `json:"readme"` I18N []string `json:"i18n"` Funding *Funding `json:"funding"` } type StageRepo struct { URL string `json:"url"` Updated string `json:"updated"` Stars int `json:"stars"` OpenIssues int `json:"openIssues"` Size int64 `json:"size"` InstallSize int64 `json:"installSize"` Package *StagePackage `json:"package"` } type StageIndex struct { Repos []*StageRepo `json:"repos"` } func getPreferredReadme(readme *Readme) string { if nil == readme { return "README.md" } ret := readme.Default switch util.Lang { case "zh_CN": if "" != readme.ZhCN { ret = readme.ZhCN } case "zh_CHT": if "" != readme.ZhCHT { ret = readme.ZhCHT } else if "" != readme.ZhCN { ret = readme.ZhCN } case "en_US": if "" != readme.EnUS { ret = readme.EnUS } default: if "" != readme.EnUS { ret = readme.EnUS } } return ret } func GetPreferredName(pkg *Package) string { if nil == pkg.DisplayName { return pkg.Name } ret := pkg.DisplayName.Default switch util.Lang { case "zh_CN": if "" != pkg.DisplayName.ZhCN { ret = pkg.DisplayName.ZhCN } case "zh_CHT": if "" != pkg.DisplayName.ZhCHT { ret = pkg.DisplayName.ZhCHT } else if "" != pkg.DisplayName.ZhCN { ret = pkg.DisplayName.ZhCN } case "en_US": if "" != pkg.DisplayName.EnUS { ret = pkg.DisplayName.EnUS } default: if "" != pkg.DisplayName.EnUS { ret = pkg.DisplayName.EnUS } } return ret } func getPreferredDesc(desc *Description) string { if nil == desc { return "" } ret := desc.Default switch util.Lang { case "zh_CN": if "" != desc.ZhCN { ret = desc.ZhCN } case "zh_CHT": if "" != desc.ZhCHT { ret = desc.ZhCHT } else if "" != desc.ZhCN { ret = desc.ZhCN } case "en_US": if "" != desc.EnUS { ret = desc.EnUS } default: if "" != desc.EnUS { ret = desc.EnUS } } return ret } func getPreferredFunding(funding *Funding) string { if nil == funding { return "" } if "" != funding.OpenCollective { return "https://opencollective.com/" + funding.OpenCollective } if "" != funding.Patreon { return "https://www.patreon.com/" + funding.Patreon } if "" != funding.GitHub { return "https://github.com/sponsors/" + funding.GitHub } if 0 < len(funding.Custom) { return funding.Custom[0] } return "" } func PluginJSON(pluginDirName string) (ret *Plugin, err error) { p := filepath.Join(util.DataDir, "plugins", pluginDirName, "plugin.json") if !filelock.IsExist(p) { err = os.ErrNotExist return } data, err := filelock.ReadFile(p) if err != nil { logging.LogErrorf("read plugin.json [%s] failed: %s", p, err) return } if err = gulu.JSON.UnmarshalJSON(data, &ret); err != nil { logging.LogErrorf("parse plugin.json [%s] failed: %s", p, err) return } ret.URL = strings.TrimSuffix(ret.URL, "/") return } func WidgetJSON(widgetDirName string) (ret *Widget, err error) { p := filepath.Join(util.DataDir, "widgets", widgetDirName, "widget.json") if !filelock.IsExist(p) { err = os.ErrNotExist return } data, err := filelock.ReadFile(p) if err != nil { logging.LogErrorf("read widget.json [%s] failed: %s", p, err) return } if err = gulu.JSON.UnmarshalJSON(data, &ret); err != nil { logging.LogErrorf("parse widget.json [%s] failed: %s", p, err) return } ret.URL = strings.TrimSuffix(ret.URL, "/") return } func IconJSON(iconDirName string) (ret *Icon, err error) { p := filepath.Join(util.IconsPath, iconDirName, "icon.json") if !gulu.File.IsExist(p) { err = os.ErrNotExist return } data, err := os.ReadFile(p) if err != nil { logging.LogErrorf("read icon.json [%s] failed: %s", p, err) return } if err = gulu.JSON.UnmarshalJSON(data, &ret); err != nil { logging.LogErrorf("parse icon.json [%s] failed: %s", p, err) return } ret.URL = strings.TrimSuffix(ret.URL, "/") return } func TemplateJSON(templateDirName string) (ret *Template, err error) { p := filepath.Join(util.DataDir, "templates", templateDirName, "template.json") if !filelock.IsExist(p) { err = os.ErrNotExist return } data, err := filelock.ReadFile(p) if err != nil { logging.LogErrorf("read template.json [%s] failed: %s", p, err) return } if err = gulu.JSON.UnmarshalJSON(data, &ret); err != nil { logging.LogErrorf("parse template.json [%s] failed: %s", p, err) return } ret.URL = strings.TrimSuffix(ret.URL, "/") return } func ThemeJSON(themeDirName string) (ret *Theme, err error) { p := filepath.Join(util.ThemesPath, themeDirName, "theme.json") if !gulu.File.IsExist(p) { err = os.ErrNotExist return } data, err := os.ReadFile(p) if err != nil { logging.LogErrorf("read theme.json [%s] failed: %s", p, err) return } ret = &Theme{} if err = gulu.JSON.UnmarshalJSON(data, &ret); err != nil { logging.LogErrorf("parse theme.json [%s] failed: %s", p, err) return } ret.URL = strings.TrimSuffix(ret.URL, "/") return } var cachedStageIndex = map[string]*StageIndex{} var stageIndexCacheTime int64 var stageIndexLock = sync.Mutex{} func getStageIndex(pkgType string) (ret *StageIndex, err error) { rhyRet, err := util.GetRhyResult(false) if err != nil { return } stageIndexLock.Lock() defer stageIndexLock.Unlock() now := time.Now().Unix() if 3600 >= now-stageIndexCacheTime && nil != cachedStageIndex[pkgType] { ret = cachedStageIndex[pkgType] return } bazaarHash := rhyRet["bazaar"].(string) ret = &StageIndex{} request := httpclient.NewBrowserRequest() u := util.BazaarOSSServer + "/bazaar@" + bazaarHash + "/stage/" + pkgType + ".json" resp, reqErr := request.SetSuccessResult(ret).Get(u) if nil != reqErr { logging.LogErrorf("get community stage index [%s] failed: %s", u, reqErr) return } if 200 != resp.StatusCode { logging.LogErrorf("get community stage index [%s] failed: %d", u, resp.StatusCode) return } stageIndexCacheTime = now cachedStageIndex[pkgType] = ret return } func isOutdatedTheme(theme *Theme, bazaarThemes []*Theme) bool { if !strings.HasPrefix(theme.URL, "https://github.com/") { return false } repo := strings.TrimPrefix(theme.URL, "https://github.com/") parts := strings.Split(repo, "/") if 2 != len(parts) || "" == strings.TrimSpace(parts[1]) { return false } for _, pkg := range bazaarThemes { if theme.URL == pkg.URL && theme.Name == pkg.Name && theme.Author == pkg.Author && 0 > semver.Compare("v"+theme.Version, "v"+pkg.Version) { theme.RepoHash = pkg.RepoHash return true } } return false } func isOutdatedIcon(icon *Icon, bazaarIcons []*Icon) bool { if !strings.HasPrefix(icon.URL, "https://github.com/") { return false } repo := strings.TrimPrefix(icon.URL, "https://github.com/") parts := strings.Split(repo, "/") if 2 != len(parts) || "" == strings.TrimSpace(parts[1]) { return false } for _, pkg := range bazaarIcons { if icon.URL == pkg.URL && icon.Name == pkg.Name && icon.Author == pkg.Author && 0 > semver.Compare("v"+icon.Version, "v"+pkg.Version) { icon.RepoHash = pkg.RepoHash return true } } return false } func isOutdatedPlugin(plugin *Plugin, bazaarPlugins []*Plugin) bool { if !strings.HasPrefix(plugin.URL, "https://github.com/") { return false } repo := strings.TrimPrefix(plugin.URL, "https://github.com/") parts := strings.Split(repo, "/") if 2 != len(parts) || "" == strings.TrimSpace(parts[1]) { return false } for _, pkg := range bazaarPlugins { if plugin.URL == pkg.URL && plugin.Name == pkg.Name && plugin.Author == pkg.Author && 0 > semver.Compare("v"+plugin.Version, "v"+pkg.Version) { plugin.RepoHash = pkg.RepoHash return true } } return false } func isOutdatedWidget(widget *Widget, bazaarWidgets []*Widget) bool { if !strings.HasPrefix(widget.URL, "https://github.com/") { return false } repo := strings.TrimPrefix(widget.URL, "https://github.com/") parts := strings.Split(repo, "/") if 2 != len(parts) || "" == strings.TrimSpace(parts[1]) { return false } for _, pkg := range bazaarWidgets { if widget.URL == pkg.URL && widget.Name == pkg.Name && widget.Author == pkg.Author && 0 > semver.Compare("v"+widget.Version, "v"+pkg.Version) { widget.RepoHash = pkg.RepoHash return true } } return false } func isOutdatedTemplate(template *Template, bazaarTemplates []*Template) bool { if !strings.HasPrefix(template.URL, "https://github.com/") { return false } repo := strings.TrimPrefix(template.URL, "https://github.com/") parts := strings.Split(repo, "/") if 2 != len(parts) || "" == strings.TrimSpace(parts[1]) { return false } for _, pkg := range bazaarTemplates { if template.URL == pkg.URL && template.Name == pkg.Name && template.Author == pkg.Author && 0 > semver.Compare("v"+template.Version, "v"+pkg.Version) { template.RepoHash = pkg.RepoHash return true } } return false } func isBazzarOnline() (ret bool) { // Improve marketplace loading when offline https://github.com/siyuan-note/siyuan/issues/12050 ret = util.IsOnline(util.BazaarOSSServer, true) if !ret { util.PushErrMsg(util.Langs[util.Lang][24], 5000) } return } func GetPackageREADME(repoURL, repoHash, packageType string) (ret string) { repoURLHash := repoURL + "@" + repoHash stageIndex := cachedStageIndex[packageType] if nil == stageIndex { return } url := strings.TrimPrefix(repoURLHash, "https://github.com/") var repo *StageRepo for _, r := range stageIndex.Repos { if r.URL == url { repo = r break } } if nil == repo { return } readme := getPreferredReadme(repo.Package.Readme) data, err := downloadPackage(repoURLHash+"/"+readme, false, "") if err != nil { ret = fmt.Sprintf("Load bazaar package's README.md(%s) failed: %s", readme, err.Error()) if readme == repo.Package.Readme.Default || "" == strings.TrimSpace(repo.Package.Readme.Default) { return } readme = repo.Package.Readme.Default data, err = downloadPackage(repoURLHash+"/"+readme, false, "") if err != nil { ret += fmt.Sprintf("
Load bazaar package's README.md(%s) failed: %s", readme, err.Error()) return } } if 2 < len(data) { if 255 == data[0] && 254 == data[1] { data, _, err = transform.Bytes(textUnicode.UTF16(textUnicode.LittleEndian, textUnicode.ExpectBOM).NewDecoder(), data) } else if 254 == data[0] && 255 == data[1] { data, _, err = transform.Bytes(textUnicode.UTF16(textUnicode.BigEndian, textUnicode.ExpectBOM).NewDecoder(), data) } } ret, err = renderREADME(repoURL, data) return } func renderREADME(repoURL string, mdData []byte) (ret string, err error) { luteEngine := lute.New() luteEngine.SetSoftBreak2HardBreak(false) luteEngine.SetCodeSyntaxHighlight(false) linkBase := "https://cdn.jsdelivr.net/gh/" + strings.TrimPrefix(repoURL, "https://github.com/") luteEngine.SetLinkBase(linkBase) ret = luteEngine.Md2HTML(string(mdData)) ret = util.LinkTarget(ret, linkBase) return } var ( packageLocks = map[string]*sync.Mutex{} packageLocksLock = sync.Mutex{} ) func downloadPackage(repoURLHash string, pushProgress bool, systemID string) (data []byte, err error) { packageLocksLock.Lock() defer packageLocksLock.Unlock() // repoURLHash: https://github.com/88250/Comfortably-Numb@6286912c381ef3f83e455d06ba4d369c498238dc repoURL := repoURLHash[:strings.LastIndex(repoURLHash, "@")] lock, ok := packageLocks[repoURLHash] if !ok { lock = &sync.Mutex{} packageLocks[repoURLHash] = lock } lock.Lock() defer lock.Unlock() repoURLHash = strings.TrimPrefix(repoURLHash, "https://github.com/") u := util.BazaarOSSServer + "/package/" + repoURLHash buf := &bytes.Buffer{} resp, err := httpclient.NewCloudFileRequest2m().SetOutput(buf).SetDownloadCallback(func(info req.DownloadInfo) { if pushProgress { progress := float32(info.DownloadedSize) / float32(info.Response.ContentLength) //logging.LogDebugf("downloading bazaar package [%f]", progress) util.PushDownloadProgress(repoURL, progress) } }).Get(u) if err != nil { logging.LogErrorf("get bazaar package [%s] failed: %s", u, err) return nil, errors.New("get bazaar package failed, please check your network") } if 200 != resp.StatusCode { logging.LogErrorf("get bazaar package [%s] failed: %d", u, resp.StatusCode) return nil, errors.New("get bazaar package failed: " + resp.Status) } data = buf.Bytes() go incPackageDownloads(repoURLHash, systemID) return } func incPackageDownloads(repoURLHash, systemID string) { if strings.Contains(repoURLHash, ".md") || "" == systemID { return } repo := strings.Split(repoURLHash, "@")[0] u := util.GetCloudServer() + "/apis/siyuan/bazaar/addBazaarPackageDownloadCount" httpclient.NewCloudRequest30s().SetBody( map[string]interface{}{ "systemID": systemID, "repo": repo, }).Post(u) } func uninstallPackage(installPath string) (err error) { if err = os.RemoveAll(installPath); err != nil { logging.LogErrorf("remove [%s] failed: %s", installPath, err) return fmt.Errorf("remove community package [%s] failed", filepath.Base(installPath)) } packageCache.Flush() return } func installPackage(data []byte, installPath, repoURLHash string) (err error) { err = installPackage0(data, installPath) if err != nil { return } packageCache.Delete(strings.TrimPrefix(repoURLHash, "https://github.com/")) return } func installPackage0(data []byte, installPath string) (err error) { tmpPackage := filepath.Join(util.TempDir, "bazaar", "package") if err = os.MkdirAll(tmpPackage, 0755); err != nil { return } name := gulu.Rand.String(7) tmp := filepath.Join(tmpPackage, name+".zip") if err = os.WriteFile(tmp, data, 0644); err != nil { return } unzipPath := filepath.Join(tmpPackage, name) if err = gulu.Zip.Unzip(tmp, unzipPath); err != nil { logging.LogErrorf("write file [%s] failed: %s", installPath, err) return } dirs, err := os.ReadDir(unzipPath) if err != nil { return } srcPath := unzipPath if 1 == len(dirs) && dirs[0].IsDir() { srcPath = filepath.Join(unzipPath, dirs[0].Name()) } if err = filelock.Copy(srcPath, installPath); err != nil { return } return } func formatUpdated(updated string) (ret string) { t, e := dateparse.ParseIn(updated, time.Now().Location()) if nil == e { ret = t.Format("2006-01-02") } else { if strings.Contains(updated, "T") { ret = updated[:strings.Index(updated, "T")] } else { ret = strings.ReplaceAll(strings.ReplaceAll(updated, "T", ""), "Z", "") } } return } type bazaarPackage struct { Name string `json:"name"` Downloads int `json:"downloads"` } var cachedBazaarIndex = map[string]*bazaarPackage{} var bazaarIndexCacheTime int64 var bazaarIndexLock = sync.Mutex{} func getBazaarIndex() map[string]*bazaarPackage { bazaarIndexLock.Lock() defer bazaarIndexLock.Unlock() now := time.Now().Unix() if 3600 >= now-bazaarIndexCacheTime { return cachedBazaarIndex } request := httpclient.NewBrowserRequest() u := util.BazaarStatServer + "/bazaar/index.json" resp, reqErr := request.SetSuccessResult(&cachedBazaarIndex).Get(u) if nil != reqErr { logging.LogErrorf("get bazaar index [%s] failed: %s", u, reqErr) return cachedBazaarIndex } if 200 != resp.StatusCode { logging.LogErrorf("get bazaar index [%s] failed: %d", u, resp.StatusCode) return cachedBazaarIndex } bazaarIndexCacheTime = now return cachedBazaarIndex } // defaultMinAppVersion 如果集市包中缺失 minAppVersion 项,则使用该值作为最低支持的版本号,小于该版本号时不显示集市包 // Add marketplace package config item `minAppVersion` https://github.com/siyuan-note/siyuan/issues/8330 const defaultMinAppVersion = "2.9.0" func disallowDisplayBazaarPackage(pkg *Package) bool { if "" == pkg.MinAppVersion { // TODO: 目前暂时放过所有不带 minAppVersion 的集市包,后续版本会使用 defaultMinAppVersion return false } if 0 < semver.Compare("v"+pkg.MinAppVersion, "v"+util.Ver) { return true } return false } var packageCache = gcache.New(6*time.Hour, 30*time.Minute) // [repoURL]*Package var packageInstallSizeCache = gcache.New(48*time.Hour, 6*time.Hour) // [repoURL]*int64