Merge pull request #47 from achilleas-k/gindex-new-query

New gin-dex query format and encrypted payloads

cmd/admin.go

@@ -30,6 +30,7 @@ to make automatic initialization process more smoothly`,
 			subcmdRewriteAuthorizedKeys,
 			subcmdSyncRepositoryHooks,
 			subcmdReinitMissingRepositories,
+			subcmdRebuildSearchIndex,
 		},
 	}
 

cmd/admin_gin.go

@@ -0,0 +1,25 @@
+// Copyright 2016 The Gogs Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package cmd
+
+import (
+	"github.com/urfave/cli"
+
+	"github.com/G-Node/gogs/models"
+)
+
+var (
+	subcmdRebuildSearchIndex = cli.Command{
+		Name:  "rebuild-index",
+		Usage: "Rebuild the search index for all repositories",
+		Action: adminDashboardOperation(
+			models.RebuildIndex,
+			"Sending all existing repositories to the gin-dex server for reindexing",
+		),
+		Flags: []cli.Flag{
+			stringFlag("config, c", "custom/conf/app.ini", "Custom configuration file path"),
+		},
+	}
+)

cmd/serv.go

@@ -273,8 +273,8 @@ func runServ(c *cli.Context) error {
 		cmd = []string{verb, repoFullName}
 	}
 	runGit(cmd, requestMode, user, owner, repo)
-	if setting.Search.Do && (requestMode == models.ACCESS_MODE_WRITE) {
-		models.StartIndexing(user, owner, repo)
+	if requestMode == models.ACCESS_MODE_WRITE {
+		models.StartIndexing(*repo)
 	}
 	return nil
 

cmd/web.go

@@ -201,6 +201,7 @@ func runWeb(c *cli.Context) error {
 		m.Get("/repos", routes.ExploreRepos)
 		m.Get("/users", routes.ExploreUsers)
 		m.Get("/organizations", routes.ExploreOrganizations)
+		m.Get("/_suggest/:keywords", routes.ExploreSuggest)
 	}, ignSignIn)
 	m.Combo("/install", routes.InstallInit).Get(routes.Install).
 		Post(bindIgnErr(form.Install{}), routes.InstallPost)

gogs.go

@@ -16,7 +16,7 @@ import (
 	"github.com/G-Node/gogs/pkg/setting"
 )
 
-const APP_VER = "0.11.87.0206"
+const APP_VER = "0.11.87.0206.gin0001"
 
 func init() {
 	setting.AppVer = APP_VER

models/models_gin.go

@@ -1,39 +1,73 @@
 package models
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
 	"net/http"
 	"os"
 	"path/filepath"
+	"strings"
 
 	gannex "github.com/G-Node/go-annex"
 	"github.com/G-Node/gogs/pkg/setting"
+	"github.com/G-Node/libgin/libgin"
 	log "gopkg.in/clog.v1"
 )
 
 // StartIndexing sends an indexing request to the configured indexing service
 // for a repository.
-func StartIndexing(user, owner *User, repo *Repository) {
-	if !setting.Search.Do {
-		return
+func StartIndexing(repo Repository) {
+	go func() {
+		if setting.Search.IndexURL == "" {
+			log.Trace("Indexing not enabled")
+			return
+		}
+		log.Trace("Indexing repository %d", repo.ID)
+		ireq := libgin.IndexRequest{
+			RepoID:   repo.ID,
+			RepoPath: repo.FullName(),
+		}
+		data, err := json.Marshal(ireq)
+		if err != nil {
+			log.Error(2, "Could not marshal index request: %v", err)
+			return
+		}
+		key := []byte(setting.Search.Key)
+		encdata, err := libgin.EncryptString(key, string(data))
+		if err != nil {
+			log.Error(2, "Could not encrypt index request: %v", err)
+		}
+		req, err := http.NewRequest(http.MethodPost, setting.Search.IndexURL, strings.NewReader(encdata))
+		if err != nil {
+			log.Error(2, "Error creating index request")
+		}
+		client := http.Client{}
+		resp, err := client.Do(req)
+		if err != nil || resp.StatusCode != http.StatusOK {
+			log.Error(2, "Error submitting index request for [%d: %s]: %v", repo.ID, repo.FullName(), err)
+			return
+		}
+	}()
+}
+
+// RebuildIndex sends all repositories to the indexing service to be indexed.
+func RebuildIndex() error {
+	indexurl := setting.Search.IndexURL
+	if indexurl == "" {
+		return fmt.Errorf("Indexing service not configured")
 	}
-	var ireq struct{ RepoID, RepoPath string }
-	ireq.RepoID = fmt.Sprintf("%d", repo.ID)
-	ireq.RepoPath = repo.FullName()
-	data, err := json.Marshal(ireq)
-	if err != nil {
-		log.Trace("could not marshal index request :%+v", err)
-		return
+
+	// collect all repo ID -> Path mappings directly from the DB
+	repos := make(RepositoryList, 0, 100)
+	if err := x.Find(&repos); err != nil {
+		return fmt.Errorf("get all repos: %v", err)
 	}
-	req, _ := http.NewRequest(http.MethodPost, setting.Search.IndexURL, bytes.NewReader(data))
-	client := http.Client{}
-	resp, err := client.Do(req)
-	if err != nil || resp.StatusCode != http.StatusOK {
-		log.Trace("Error doing index request:%+v", err)
-		return
+	log.Trace("Found %d repositories to index", len(repos))
+	for _, repo := range repos {
+		StartIndexing(*repo)
 	}
+	log.Trace("Rebuilding search index")
+	return nil
 }
 
 func annexUninit(path string) {
@@ -64,7 +98,7 @@ func annexUninit(path string) {
 	}
 }
 
-func annexAdd(path string) {
+func annexSetup(path string) {
 	log.Trace("Running annex add (with filesize filter) in '%s'", path)
 
 	// Initialise annex in case it's a new repository
@@ -83,9 +117,9 @@ func annexAdd(path string) {
 		log.Error(1, "Failed to set default backend to 'MD5': %v (%s)", err, msg)
 	}
 
-	sizefilterflag := fmt.Sprintf("--largerthan=%d", setting.Repository.Upload.AnnexFileMinSize*gannex.MEGABYTE)
-	if msg, err := gannex.Add(path, sizefilterflag); err != nil {
-		log.Error(1, "Annex add failed with error: %v (%s)", err, msg)
+	// Set size filter in config
+	if msg, err := gannex.SetAnnexSizeFilter(path, setting.Repository.Upload.AnnexFileMinSize*gannex.MEGABYTE); err != nil {
+		log.Error(2, "Failed to set size filter for annex: %v (%s)", err, msg)
 	}
 }
 

models/repo_editor.go

@@ -194,9 +194,7 @@ func (repo *Repository) UpdateRepoFile(doer *User, opts UpdateRepoFileOptions) (
 		return fmt.Errorf("git push origin %s: %v", opts.NewBranch, err)
 	}
 
-	if setting.Search.Do {
-		StartIndexing(doer, repo.MustOwner(), repo)
-	}
+	StartIndexing(*repo)
 	return nil
 }
 
@@ -499,7 +497,7 @@ func (repo *Repository) UploadRepoFiles(doer *User, opts UploadRepoFileOptions)
 		}
 	}
 
-	annexAdd(localPath) // Running annex add with filter for big files
+	annexSetup(localPath) // Initialise annex and set configuration (with add filter for filesizes)
 	if err = git.AddChanges(localPath, true); err != nil {
 		return fmt.Errorf("git add --all: %v", err)
 	} else if err = git.CommitChanges(localPath, git.CommitChangesOptions{
@@ -522,7 +520,7 @@ func (repo *Repository) UploadRepoFiles(doer *User, opts UploadRepoFileOptions)
 	if err := annexSync(localPath); err != nil { // Run full annex sync
 		return err
 	}
-	annexUninit(localPath)                      // Uninitialise annex to prepare for deletion
-	StartIndexing(doer, repo.MustOwner(), repo) // Index the new data
+	annexUninit(localPath) // Uninitialise annex to prepare for deletion
+	StartIndexing(*repo)   // Index the new data
 	return DeleteUploads(uploads...)
 }

pkg/mailer/mail.go

@@ -52,7 +52,7 @@ func InitMailRender(dir, appendDir string, funcMap []template.FuncMap) {
 }
 
 func SendTestMail(email string) error {
-	return gomail.Send(&Sender{}, NewMessage([]string{email}, "Gogs Test Email!", "Gogs Test Email!").Message)
+	return gomail.Send(&Sender{}, NewMessage(email, "Gogs Test Email!", "Gogs Test Email!").Message)
 }
 
 /*
@@ -95,7 +95,7 @@ func SendUserMail(c *macaron.Context, u User, tpl, code, subject, info string) {
 		return
 	}
 
-	msg := NewMessage([]string{u.Email()}, subject, body)
+	msg := NewMessage(u.Email(), subject, body)
 	msg.Info = fmt.Sprintf("UID: %d, %s", u.ID(), info)
 
 	Send(msg)
@@ -127,7 +127,7 @@ func SendActivateEmailMail(c *macaron.Context, u User, email string) {
 		return
 	}
 
-	msg := NewMessage([]string{email}, c.Tr("mail.activate_email"), body)
+	msg := NewMessage(email, c.Tr("mail.activate_email"), body)
 	msg.Info = fmt.Sprintf("UID: %d, activate email", u.ID())
 
 	Send(msg)
@@ -144,7 +144,7 @@ func SendRegisterNotifyMail(c *macaron.Context, u User) {
 		return
 	}
 
-	msg := NewMessage([]string{u.Email()}, c.Tr("mail.register_notify"), body)
+	msg := NewMessage(u.Email(), c.Tr("mail.register_notify"), body)
 	msg.Info = fmt.Sprintf("UID: %d, registration notify", u.ID())
 
 	Send(msg)
@@ -165,7 +165,7 @@ func SendCollaboratorMail(u, doer User, repo Repository) {
 		return
 	}
 
-	msg := NewMessage([]string{u.Email()}, subject, body)
+	msg := NewMessage(u.Email(), subject, body)
 	msg.Info = fmt.Sprintf("UID: %d, add collaborator", u.ID())
 
 	Send(msg)
@@ -179,7 +179,7 @@ func composeTplData(subject, body, link string) map[string]interface{} {
 	return data
 }
 
-func composeIssueMessage(issue Issue, repo Repository, doer User, tplName string, tos []string, info string) *Message {
+func composeIssueMessages(issue Issue, repo Repository, doer User, tplName string, tos []string, info string) []*Message {
 	subject := issue.MailSubject()
 	body := string(markup.Markdown([]byte(issue.Content()), repo.HTMLURL(), repo.ComposeMetas()))
 	data := composeTplData(subject, body, issue.HTMLURL())
@@ -189,9 +189,13 @@ func composeIssueMessage(issue Issue, repo Repository, doer User, tplName string
 		log.Error(3, "HTMLString (%s): %v", tplName, err)
 	}
 	from := gomail.NewMessage().FormatAddress(setting.MailService.FromEmail, doer.DisplayName())
-	msg := NewMessageFrom(tos, from, subject, content)
-	msg.Info = fmt.Sprintf("Subject: %s, %s", subject, info)
-	return msg
+	msgs := make([]*Message, len(tos))
+	for idx, to := range tos {
+		msg := NewMessageFrom(to, from, subject, content)
+		msg.Info = fmt.Sprintf("Subject: %s, %s", subject, info)
+		msgs[idx] = msg
+	}
+	return msgs
 }
 
 // SendIssueCommentMail composes and sends issue comment emails to target receivers.
@@ -199,8 +203,10 @@ func SendIssueCommentMail(issue Issue, repo Repository, doer User, tos []string)
 	if len(tos) == 0 {
 		return
 	}
-
-	Send(composeIssueMessage(issue, repo, doer, MAIL_ISSUE_COMMENT, tos, "issue comment"))
+	msgs := composeIssueMessages(issue, repo, doer, MAIL_ISSUE_COMMENT, tos, "issue comment")
+	for _, msg := range msgs {
+		Send(msg)
+	}
 }
 
 // SendIssueMentionMail composes and sends issue mention emails to target receivers.
@@ -208,5 +214,8 @@ func SendIssueMentionMail(issue Issue, repo Repository, doer User, tos []string)
 	if len(tos) == 0 {
 		return
 	}
-	Send(composeIssueMessage(issue, repo, doer, MAIL_ISSUE_MENTION, tos, "issue mention"))
+	msgs := composeIssueMessages(issue, repo, doer, MAIL_ISSUE_MENTION, tos, "issue mention")
+	for _, msg := range msgs {
+		Send(msg)
+	}
 }

pkg/mailer/mailer.go

@@ -28,12 +28,12 @@ type Message struct {
 }
 
 // NewMessageFrom creates new mail message object with custom From header.
-func NewMessageFrom(to []string, from, subject, htmlBody string) *Message {
+func NewMessageFrom(to string, from, subject, htmlBody string) *Message {
 	log.Trace("NewMessageFrom (htmlBody):\n%s", htmlBody)
 
 	msg := gomail.NewMessage()
 	msg.SetHeader("From", from)
-	msg.SetHeader("To", to...)
+	msg.SetHeader("To", to)
 	msg.SetHeader("Subject", setting.MailService.SubjectPrefix+subject)
 	msg.SetDateHeader("Date", time.Now())
 
@@ -64,7 +64,7 @@ func NewMessageFrom(to []string, from, subject, htmlBody string) *Message {
 }
 
 // NewMessage creates new mail message object with default From header.
-func NewMessage(to []string, subject, body string) *Message {
+func NewMessage(to string, subject, body string) *Message {
 	return NewMessageFrom(to, setting.MailService.From, subject, body)
 }
 

pkg/setting/setting.go

@@ -331,9 +331,9 @@ var (
 	HasRobotsTxt bool
 
 	Search struct {
-		Do        bool
-		IndexURL  string
-		SearchURL string
+		IndexURL  string `ini:"INDEX_URL"`
+		SearchURL string `ini:"SEARCH_URL"`
+		Key       string `ini:"SEARCH_KEY"`
 	}
 
 	DOI struct {
@@ -532,7 +532,7 @@ func NewContext() {
 	SSH.ServerCiphers = sec.Key("SSH_SERVER_CIPHERS").Strings(",")
 	SSH.KeyTestPath = os.TempDir()
 	if err = Cfg.Section("server").MapTo(&SSH); err != nil {
-		log.Fatal(2, "Fail to map SSH settings: %v", err)
+		log.Fatal(2, "Failed to map SSH settings: %v", err)
 	}
 	if SSH.Disabled {
 		SSH.StartBuiltinServer = false
@@ -632,11 +632,11 @@ func NewContext() {
 	}
 	ScriptType = sec.Key("SCRIPT_TYPE").MustString("bash")
 	if err = Cfg.Section("repository").MapTo(&Repository); err != nil {
-		log.Fatal(2, "Fail to map Repository settings: %v", err)
+		log.Fatal(2, "Failed to map Repository settings: %v", err)
 	} else if err = Cfg.Section("repository.editor").MapTo(&Repository.Editor); err != nil {
-		log.Fatal(2, "Fail to map Repository.Editor settings: %v", err)
+		log.Fatal(2, "Failed to map Repository.Editor settings: %v", err)
 	} else if err = Cfg.Section("repository.upload").MapTo(&Repository.Upload); err != nil {
-		log.Fatal(2, "Fail to map Repository.Upload settings: %v", err)
+		log.Fatal(2, "Failed to map Repository.Upload settings: %v", err)
 	}
 
 	if !filepath.IsAbs(Repository.Upload.TempPath) {
@@ -713,13 +713,13 @@ func NewContext() {
 	} else if err = Cfg.Section("prometheus").MapTo(&Prometheus); err != nil {
 		log.Fatal(2, "Failed to map Prometheus settings: %v", err)
 	} else if err = Cfg.Section("search").MapTo(&Search); err != nil {
-		log.Fatal(2, "Fail to map Search settings: %v", err)
+		log.Fatal(2, "Failed to map Search settings: %v", err)
 	} else if err = Cfg.Section("doi").MapTo(&DOI); err != nil {
-		log.Fatal(2, "Fail to map DOI settings: %v", err)
+		log.Fatal(2, "Failed to map DOI settings: %v", err)
 	} else if err = Cfg.Section("cliconfig").MapTo(&CLIConfig); err != nil {
-		log.Fatal(2, "Fail to map Client config settings: %v", err)
+		log.Fatal(2, "Failed to map Client config settings: %v", err)
 	} else if err = Cfg.Section("dav").MapTo(&WebDav); err != nil {
-		log.Fatal(2, "Fail to map WebDav settings: %v", err)
+		log.Fatal(2, "Failed to map WebDav settings: %v", err)
 	}
 
 	if Mirror.DefaultInterval <= 0 {

public/js/gogs.js

@@ -1152,11 +1152,11 @@ function initSuggest() {
     $('.ui.ginsearch')
         .search({
             apiSettings: {
-                url: '/api/v1/repos/suggest/{query}'
+                url: '/explore/_suggest/{query}'
             },
             fields: {
                 results: 'Items',
-                title: 'Title'
+                title: 'Keyword',
             },
             minCharacters: 3,
             showNoResults: false

routes/admin/admin.go

@@ -11,7 +11,7 @@ import (
 	"time"
 
 	"github.com/Unknwon/com"
-	"github.com/json-iterator/go"
+	jsoniter "github.com/json-iterator/go"
 	"gopkg.in/macaron.v1"
 
 	"github.com/G-Node/gogs/models"
@@ -123,6 +123,7 @@ const (
 	SYNC_SSH_AUTHORIZED_KEY
 	SYNC_REPOSITORY_HOOKS
 	REINIT_MISSING_REPOSITORY
+	REBUILD_SEARCH_INDEX
 )
 
 func Dashboard(c *context.Context) {
@@ -158,6 +159,10 @@ func Dashboard(c *context.Context) {
 		case REINIT_MISSING_REPOSITORY:
 			success = c.Tr("admin.dashboard.reinit_missing_repos_success")
 			err = models.ReinitMissingRepositories()
+		case REBUILD_SEARCH_INDEX:
+			// TODO: Add success message to locale files
+			success = "All repositories have been submitted to the indexing service successfully."
+			err = models.RebuildIndex()
 		}
 
 		if err != nil {

routes/api/v1/repo/repo.go

@@ -69,14 +69,12 @@ func Search(c *context.APIContext) {
 		return
 	}
 
-	results := make([]*api.Repository, len(repos))
+	results := make([]*api.Repository, 0, len(repos))
 	for i := range repos {
 		if !repos[i].Unlisted {
 			rep := repos[i].APIFormat(nil)
-			if !c.IsLogged {
-				rep.Owner.Email = ""
-			}
-			results[i] = rep
+			rep.Owner.Email = ""
+			results = append(results, rep)
 		}
 	}
 

routes/api/v1/search/general.go

@@ -12,16 +12,16 @@ import (
 )
 
 func Search(c *context.APIContext) {
+	// TODO: API calls shouldn't use cookie (see https://github.com/G-Node/gin-dex/issues/2)
 	if !c.IsLogged {
 		c.Status(http.StatusUnauthorized)
 		return
 	}
-	if !setting.Search.Do {
+	if setting.Search.SearchURL == "" {
 		c.Status(http.StatusNotImplemented)
 		return
 	}
-	ireq := libgin.SearchRequest{Token: c.GetCookie(setting.SessionConfig.CookieName), UserID: c.User.ID,
-		Query: c.Params("query"), CsrfT: c.GetCookie(setting.CSRFCookieName)}
+	ireq := libgin.SearchRequest{}
 	data, err := json.Marshal(ireq)
 	if err != nil {
 		c.Status(http.StatusInternalServerError)

routes/api/v1/search/suggest.go

@@ -12,15 +12,12 @@ import (
 )
 
 func Suggest(c *context.APIContext) {
-	if !setting.Search.Do {
+	// TODO: API calls shouldn't use cookie (see https://github.com/G-Node/gin-dex/issues/2)
+	if setting.Search.SearchURL == "" {
 		c.Status(http.StatusNotImplemented)
 		return
 	}
-	ireq := libgin.SearchRequest{Token: c.GetCookie(setting.SessionConfig.CookieName),
-		Query: c.Params("query"), CsrfT: c.GetCookie(setting.CSRFCookieName), SType: libgin.SEARCH_SUGGEST}
-	if c.IsLogged {
-		ireq.UserID = c.User.ID
-	}
+	ireq := libgin.SearchRequest{}
 	data, err := json.Marshal(ireq)
 	if err != nil {
 		c.Status(http.StatusInternalServerError)

routes/doi.go

@@ -1,19 +1,16 @@
 package routes
 
 import (
-	"crypto/aes"
-	"crypto/cipher"
-	"crypto/rand"
-	"encoding/base64"
-	"io"
 	"net/http"
 	"net/url"
 
 	"github.com/G-Node/gogs/pkg/context"
 	"github.com/G-Node/gogs/pkg/setting"
+	"github.com/G-Node/libgin/libgin"
 	log "gopkg.in/clog.v1"
 )
 
+// RequestDOI sends a registration request to the configured DOI service
 func RequestDOI(c *context.Context) {
 	if !c.Repo.IsAdmin() {
 		c.Status(http.StatusUnauthorized)
@@ -22,8 +19,7 @@ func RequestDOI(c *context.Context) {
 
 	repo := c.Repo.Repository.FullName()
 	username := c.User.Name
-	// verification := c.GetCookie(setting.SessionConfig.CookieName)
-	verification, err := encrypt([]byte(setting.DOI.Key), repo+username)
+	verification, err := libgin.EncryptURLString([]byte(setting.DOI.Key), repo+username)
 	if err != nil {
 		log.Error(2, "Could not encrypt secret key: %s", err)
 		c.Status(http.StatusInternalServerError)
@@ -43,29 +39,3 @@ func RequestDOI(c *context.Context) {
 	log.Trace(target)
 	c.RawRedirect(target)
 }
-
-// NOTE: TEMPORARY COPY FROM gin-doi
-
-// encrypt string to base64 crypto using AES
-func encrypt(key []byte, text string) (string, error) {
-	plaintext := []byte(text)
-
-	block, err := aes.NewCipher(key)
-	if err != nil {
-		return "", err
-	}
-
-	// The IV needs to be unique, but not secure. Therefore it's common to
-	// include it at the beginning of the ciphertext.
-	ciphertext := make([]byte, aes.BlockSize+len(plaintext))
-	iv := ciphertext[:aes.BlockSize]
-	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
-		return "", err
-	}
-
-	stream := cipher.NewCFBEncrypter(block, iv)
-	stream.XORKeyStream(ciphertext[aes.BlockSize:], plaintext)
-
-	// convert to base64
-	return base64.URLEncoding.EncodeToString(ciphertext), nil
-}

routes/repo/repo_gin.go

@@ -98,24 +98,24 @@ func calcRepoDOI(c *context.Context, doiBase string) string {
 // the two variables without modifications.
 // Any errors that occur during processing are stored in the provided context.
 // The FileSize of the annexed content is also saved in the context (c.Data["FileSize"]).
-func resolveAnnexedContent(c *context.Context, buf []byte, dataRc io.Reader) ([]byte, io.Reader) {
+func resolveAnnexedContent(c *context.Context, buf []byte, dataRc io.Reader) ([]byte, io.Reader, error) {
 	if !tool.IsAnnexedFile(buf) {
 		// not an annex pointer file; return as is
-		return buf, dataRc
+		return buf, dataRc, nil
 	}
 	log.Trace("Annexed file requested: Resolving content for [%s]", bytes.TrimSpace(buf))
 	af, err := gannex.NewAFile(c.Repo.Repository.RepoPath(), "annex", "", buf)
 	if err != nil {
 		log.Trace("Could not get annex file: %v", err)
-		c.ServerError("readmeFile.Data", err)
-		return buf, dataRc
+		c.Data["IsAnnexedFile"] = true
+		return buf, dataRc, err
 	}
 
 	afp, err := af.Open()
 	if err != nil {
-		c.ServerError("readmeFile.Data", err)
 		log.Trace("Could not open annex file: %v", err)
-		return buf, dataRc
+		c.Data["IsAnnexedFile"] = true
+		return buf, dataRc, err
 	}
 	annexDataReader := bufio.NewReader(afp)
 	annexBuf := make([]byte, 1024)
@@ -123,5 +123,5 @@ func resolveAnnexedContent(c *context.Context, buf []byte, dataRc io.Reader) ([]
 	annexBuf = annexBuf[:n]
 	c.Data["FileSize"] = af.Info.Size()
 	log.Trace("Annexed file size: %d B", af.Info.Size())
-	return annexBuf, annexDataReader
+	return annexBuf, annexDataReader, nil
 }

routes/repo/view.go

@@ -86,7 +86,10 @@ func renderDirectory(c *context.Context, treeLink string) {
 		buf = buf[:n]
 
 		// GIN mod: Replace existing buf and reader with annexed content buf and reader
-		buf, dataRc = resolveAnnexedContent(c, buf, dataRc)
+		buf, dataRc, err = resolveAnnexedContent(c, buf, dataRc)
+		if err != nil {
+			return
+		}
 
 		isTextFile := tool.IsTextFile(buf)
 		c.Data["IsTextFile"] = isTextFile
@@ -151,7 +154,10 @@ func renderFile(c *context.Context, entry *git.TreeEntry, treeLink, rawLink stri
 	buf = buf[:n]
 
 	// GIN mod: Replace existing buf and reader with annexed content buf and reader
-	buf, dataRc = resolveAnnexedContent(c, buf, dataRc)
+	buf, dataRc, err = resolveAnnexedContent(c, buf, dataRc)
+	if err != nil {
+		return
+	}
 
 	isTextFile := tool.IsTextFile(buf)
 	c.Data["IsTextFile"] = isTextFile

routes/search.go

@@ -1,13 +1,13 @@
 package routes
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/http"
-	"strconv"
+	"strings"
 
+	"github.com/G-Node/gogs/models"
 	"github.com/G-Node/gogs/pkg/context"
 	"github.com/G-Node/gogs/pkg/setting"
 	"github.com/G-Node/libgin/libgin"
@@ -19,99 +19,215 @@ const (
 	EXPLORE_COMMITS = "explore/commits"
 )
 
-func Search(c *context.Context, keywords string, sType int64) ([]byte, error) {
-	if !setting.Search.Do {
-		c.Status(http.StatusNotImplemented)
+type set map[int64]interface{}
+
+func newset() set {
+	return make(map[int64]interface{}, 0)
+}
+
+func (s set) add(item int64) {
+	s[item] = nil
+}
+
+func (s set) contains(item int64) bool {
+	_, yes := s[item]
+	return yes
+}
+
+func (s set) remove(item int64) {
+	delete(s, item)
+}
+
+func (s set) asSlice() []int64 {
+	slice := make([]int64, len(s))
+	idx := 0
+	for item := range s {
+		slice[idx] = item
+		idx++
+	}
+	return slice
+}
+
+func collectSearchableRepoIDs(c *context.Context) ([]int64, error) {
+	repoIDSet := newset()
+
+	updateSet := func(repos []*models.Repository) {
+		for _, r := range repos {
+			repoIDSet.add(r.ID)
+		}
+	}
+
+	if c.User != nil {
+		ownRepos := c.User.Repos // user's own repositories
+		updateSet(ownRepos)
+
+		accessibleRepos, _ := c.User.GetAccessibleRepositories(0) // shared and org repos
+		updateSet(accessibleRepos)
+	}
+
+	// Run a full repository search (with no keywords) to get public
+	// repositories and then filter out the unlisted ones.
+	repos, _, err := models.SearchRepositoryByName(&models.SearchRepoOptions{
+		Keyword:  "",
+		UserID:   c.UserID(),
+		OrderBy:  "updated_unix DESC",
+		Page:     0,
+		PageSize: 0,
+	})
+	if err != nil {
+		c.ServerError("SearchRepositoryByName", err)
+		return nil, err
+	}
+
+	// If it's not unlisted, add it to the set
+	// This will add public (listed) repositories
+	for _, r := range repos {
+		if !r.Unlisted {
+			repoIDSet.add(r.ID)
+		}
+	}
+
+	return repoIDSet.asSlice(), nil
+}
+
+func search(c *context.Context, keywords string, sType int) ([]byte, error) {
+	if setting.Search.SearchURL == "" {
+		log.Error(2, "Unable to perform search: SearchURL not configured")
 		return nil, fmt.Errorf("Extended search not implemented")
 	}
 
-	ireq := libgin.SearchRequest{Token: c.GetCookie(setting.SessionConfig.CookieName),
-		Query: keywords, CsrfT: c.GetCookie(setting.CSRFCookieName), SType: sType, UserID: -10}
-	if c.IsLogged {
-		ireq.UserID = c.User.ID
+	key := []byte(setting.Search.Key)
+
+	repoids, err := collectSearchableRepoIDs(c)
+	if err != nil {
+		log.Error(2, "Failed to collect searchable repository IDs: %v", err)
+		return nil, err
+	}
+	searchdata := libgin.SearchRequest{Keywords: keywords, SType: sType, RepoIDs: repoids}
+
+	data, err := json.Marshal(searchdata)
+	if err != nil {
+		log.Error(2, "Failed to marshal search request for gin-dex: %v", err)
+		return nil, err
 	}
 
-	data, err := json.Marshal(ireq)
+	// encrypt query
+	encdata, err := libgin.EncryptString(key, string(data))
 	if err != nil {
-		c.Status(http.StatusInternalServerError)
+		log.Error(2, "Failed to encrypt search data for gin-dex: %v", err)
 		return nil, err
 	}
-	req, _ := http.NewRequest("Post", setting.Search.SearchURL, bytes.NewReader(data))
+
+	// Send query to gin-dex
+	req, err := http.NewRequest("POST", setting.Search.SearchURL, strings.NewReader(encdata))
+	if err != nil {
+		log.Error(2, "Failed to build request for gin-dex: %v", err)
+	}
 	cl := http.Client{}
 	resp, err := cl.Do(req)
 	if err != nil {
-		c.Status(http.StatusInternalServerError)
+		log.Error(2, "Failed to send request to gin-dex: %v", err)
 		return nil, err
 	}
-	data, err = ioutil.ReadAll(resp.Body)
+
+	encrespdata, err := ioutil.ReadAll(resp.Body) // response is encrypted
 	if err != nil {
-		c.Status(http.StatusInternalServerError)
+		log.Error(2, "Failed to read response body from gin-dex: %v", err)
 		return nil, err
 	}
-	return data, nil
+
+	// decrypt response
+	respdata, err := libgin.DecryptString(key, string(encrespdata))
+	if err != nil {
+		log.Error(2, "Failed to decrypt response body form gin-dex: %v", err)
+		return nil, err
+	}
+
+	return []byte(respdata), nil
 }
 
+// ExploreData handles the search box served at /explore/data
 func ExploreData(c *context.Context) {
+	keywords := c.Query("q")
+	sType := c.QueryInt("stype") // non integer stype will return 0
+
 	c.Data["Title"] = c.Tr("explore")
 	c.Data["PageIsExplore"] = true
 	c.Data["PageIsExploreData"] = true
 
-	keywords := c.Query("q")
+	// send query data back even if the search fails or is aborted to fill in
+	// the form on refresh
 	c.Data["Keywords"] = keywords
-	sType, err := strconv.ParseInt(c.Query("stype"), 10, 0)
-	if err != nil {
-		log.Error(2, "Search type not understood: %s", err.Error())
-		sType = 0
+	c.Data["opsel"] = sType
+
+	res := libgin.SearchResults{}
+	if keywords == "" {
+		// no keywords submitted: don't search
+		log.Trace("Loading empty data search page")
+		c.Data["Blobs"] = res.Blobs
+		c.HTML(200, EXPLORE_DATA)
+		return
 	}
-	data, err := Search(c, keywords, sType)
+
+	log.Trace("Searching data/blobs")
+	data, err := search(c, keywords, sType)
 	if err != nil {
-		c.Handle(http.StatusInternalServerError, "Could not query", err)
+		log.Error(2, "Query returned error: %v", err)
+		c.Data["Blobs"] = res.Blobs
+		c.HTML(200, EXPLORE_DATA)
 		return
 	}
 
-	res := libgin.SearchResults{}
 	err = json.Unmarshal(data, &res)
 	if err != nil {
-		c.Handle(http.StatusInternalServerError, "Could not display result", err)
+		log.Error(2, "Failed to unmarshal response: %v", err)
+		c.Data["Blobs"] = res.Blobs
+		c.HTML(200, EXPLORE_DATA)
 		return
 	}
 	c.Data["Blobs"] = res.Blobs
-	c.Data["opsel"] = sType
 	c.HTML(200, EXPLORE_DATA)
 }
 
+// ExploreCommits handles the search box served at /explore/commits
 func ExploreCommits(c *context.Context) {
+	keywords := c.Query("q")
+	sType := c.QueryInt("stype") // non integer stype will return 0
+
 	c.Data["Title"] = c.Tr("explore")
 	c.Data["PageIsExplore"] = true
 	c.Data["PageIsExploreCommits"] = true
 
-	keywords := c.Query("q")
-	sType, err := strconv.ParseInt(c.Query("stype"), 10, 0)
-	if err != nil {
-		log.Error(2, "Search type not understood: %s", err.Error())
-		sType = 0
+	// send query data back even if the search fails or is aborted to fill in
+	// the form on refresh
+	c.Data["Keywords"] = keywords
+	c.Data["opsel"] = sType
+
+	res := libgin.SearchResults{}
+	if keywords == "" {
+		log.Trace("Loading empty commit search page")
+		// no keywords submitted: don't search
+		c.Data["Commits"] = res.Commits
+		c.HTML(200, EXPLORE_COMMITS)
+		return
 	}
-	data, err := Search(c, keywords, sType)
+
+	log.Trace("Searching commits")
+	data, err := search(c, keywords, sType)
 
 	if err != nil {
-		c.Handle(http.StatusInternalServerError, "Could not query", err)
-		return
+		log.Error(2, "Query returned error: %v", err)
+		c.Data["Commits"] = res.Commits
+		c.HTML(200, EXPLORE_COMMITS)
 	}
 
-	res := libgin.SearchResults{}
 	err = json.Unmarshal(data, &res)
 	if err != nil {
-		c.Handle(http.StatusInternalServerError, "Could not display result", err)
+		log.Error(2, "Failed to unmarshal response: %v", err)
+		c.Data["Commits"] = res.Commits
+		c.HTML(200, EXPLORE_COMMITS)
 		return
 	}
 	c.Data["Commits"] = res.Commits
 	c.HTML(200, EXPLORE_COMMITS)
 }
-
-type SearchRequest struct {
-	Token  string
-	CsrfT  string
-	UserID int64
-	Query  string
-	SType  int64
-}

routes/suggest.go

@@ -0,0 +1,36 @@
+package routes
+
+import (
+	"github.com/G-Node/gogs/pkg/context"
+	"github.com/G-Node/libgin/libgin"
+	log "gopkg.in/clog.v1"
+)
+
+// ExploreSuggest returns suggestions for keywords to fill the search box on the explore/data page.
+func ExploreSuggest(c *context.Context) {
+	keywords := c.Params(":keywords")
+	sType := libgin.SEARCH_SUGGEST
+	log.Trace("Suggestions for [%s]", keywords)
+
+	if keywords == "" {
+		// no keywords submitted: return
+		return
+	}
+	// res := libgin.SearchResults{}
+
+	log.Trace("Searching data/blobs for suggestions")
+	data, err := search(c, keywords, sType)
+	if err != nil {
+		log.Error(2, "Query returned error: %v", err)
+		return
+	}
+
+	log.Trace("Returning suggestions: %+v", string(data))
+
+	if err != nil {
+		log.Error(2, "Failed to marshal structured suggestions: %v", err)
+		return
+	}
+
+	c.Write(data)
+}

templates/.VERSION

@@ -1 +1 @@
-0.11.87.0206
+0.11.87.0206.gin0001

templates/admin/dashboard.tmpl

@@ -47,6 +47,10 @@
 								<td>{{.i18n.Tr "admin.dashboard.reinit_missing_repos"}}</td>
 								<td><i class="fa fa-caret-square-o-right"></i> <a href="{{AppSubURL}}/admin?op=7">{{.i18n.Tr "admin.dashboard.operation_run"}}</a></td>
 							</tr>
+							<tr>
+								<td>Rebuild search index</td> <!--- TODO: Add to locale files -->
+								<td><i class="fa fa-caret-square-o-right"></i> <a href="{{AppSubURL}}/admin?op=8">{{.i18n.Tr "admin.dashboard.operation_run"}}</a></td>
+							</tr>
 						</tbody>
 					</table>
 				</div>

templates/repo/header.tmpl

@@ -17,16 +17,20 @@
 						{{if .IsMirror}}<div class="fork-flag">{{$.i18n.Tr "repo.mirror_from"}} <a target="_blank" rel="noopener noreferrer" href="{{$.Mirror.Address}}">{{$.Mirror.Address}}</a></div>{{end}}
 						{{if .IsFork}}<div class="fork-flag">{{$.i18n.Tr "repo.forked_from"}} <a href="{{.BaseRepo.Link}}">{{SubStr .BaseRepo.RelLink 1 -1}}</a></div>{{end}}
 					</div>
+
 					{{if not $.IsGuest}}
 						<div class="ui right">
-							<div class="ui labeled button" tabindex="0">
-								<a class="ui basic button" href="{{$.RepoLink}}/action/{{if $.IsWatchingRepo}}un{{end}}watch?redirect_to={{$.Link}}">
-									<i class="icon fa-eye{{if not $.IsWatchingRepo}}-slash{{end}}"></i>{{if $.IsWatchingRepo}}{{$.i18n.Tr "repo.unwatch"}}{{else}}{{$.i18n.Tr "repo.watch"}}{{end}}
-								</a>
-								<a class="ui basic label" href="{{.Link}}/watchers">
-									{{.NumWatches}}
-								</a>
-							</div>
+							<form class="display inline" action="{{$.RepoLink}}/action/{{if $.IsWatchingRepo}}un{{end}}watch?redirect_to={{$.Link}}" method="POST">
+								{{$.CSRFTokenHTML}}
+								<div class="ui labeled button" tabindex="0">
+									<button class="ui basic button">
+										<i class="eye{{if not $.IsWatchingRepo}} slash outline{{end}} icon"></i>{{if $.IsWatchingRepo}}{{$.i18n.Tr "repo.unwatch"}}{{else}}{{$.i18n.Tr "repo.watch"}}{{end}}
+									</button>
+									<a class="ui basic label" href="{{.Link}}/watchers">
+										{{.NumWatches}}
+									</a>
+								</div>
+							</form>
 							{{if .CanBeForked}}
 							<div class="ui labeled button" tabindex="0"
 									 data-tooltip="Please note: annexed file content will not be forked" data-position="bottom center">

vendor/github.com/G-Node/go-annex/add.go

@@ -1,6 +1,8 @@
 package gannex
 
 import (
+	"fmt"
+
 	"github.com/G-Node/git-module"
 )
 
@@ -47,3 +49,8 @@ func SetAddUnlocked(dir string) (string, error) {
 	cmd := git.NewCommand("config", "annex.addunlocked", "true")
 	return cmd.RunInDir(dir)
 }
+
+func SetAnnexSizeFilter(dir string, size int64) (string, error) {
+	cmd := git.NewCommand("config", "annex.largefiles", fmt.Sprintf("largerthan=%d", size))
+	return cmd.RunInDir(dir)
+}

vendor/github.com/G-Node/libgin/libgin/archive.go

@@ -9,8 +9,9 @@ import (
 	"strings"
 )
 
-// MakeZip creates a zip archive called dest from the files specified by source.
-// Any directories listed in source are archived recursively.
+// MakeZip recursively writes all the files found under the provided sources to
+// the dest io.Writer in ZIP format.  Any directories listed in source are
+// archived recursively.  Empty directories are ignored.
 func MakeZip(dest io.Writer, source ...string) error {
 	// check sources
 	for _, src := range source {
@@ -22,7 +23,7 @@ func MakeZip(dest io.Writer, source ...string) error {
 	zipwriter := zip.NewWriter(dest)
 	defer zipwriter.Close()
 
-	walker := func(filepath string, fi os.FileInfo, err error) error {
+	walker := func(path string, fi os.FileInfo, err error) error {
 
 		// return on any error
 		if err != nil {
@@ -37,7 +38,7 @@ func MakeZip(dest io.Writer, source ...string) error {
 
 		// update the name to correctly reflect the desired destination when unzipping
 		// header.Name = strings.TrimPrefix(strings.Replace(file, src, "", -1), string(filepath.Separator))
-		header.Name = filepath
+		header.Name = path
 
 		if fi.Mode().IsDir() {
 			return nil
@@ -51,7 +52,7 @@ func MakeZip(dest io.Writer, source ...string) error {
 
 		// Dereference symlinks
 		if fi.Mode()&os.ModeSymlink != 0 {
-			data, err := os.Readlink(filepath)
+			data, err := os.Readlink(path)
 			if err != nil {
 				return err
 			}
@@ -62,7 +63,7 @@ func MakeZip(dest io.Writer, source ...string) error {
 		}
 
 		// open files for zipping
-		f, err := os.Open(filepath)
+		f, err := os.Open(path)
 		defer f.Close()
 		if err != nil {
 			return err

vendor/github.com/G-Node/libgin/libgin/crypto.go

@@ -0,0 +1,95 @@
+package libgin
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"encoding/base64"
+	"io"
+)
+
+// Encrypt an array of bytes with AES.
+func Encrypt(key, plaintext []byte) ([]byte, error) {
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, err
+	}
+
+	// The IV needs to be unique, but not secure. Therefore it's common to
+	// include it at the beginning of the ciphertext.
+	ciphertext := make([]byte, aes.BlockSize+len(plaintext))
+	iv := ciphertext[:aes.BlockSize]
+	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
+		return nil, err
+	}
+
+	stream := cipher.NewCFBEncrypter(block, iv)
+	stream.XORKeyStream(ciphertext[aes.BlockSize:], plaintext)
+	return ciphertext, nil
+}
+
+// EncryptURLString encrypts a string using AES and returns it in URL encoded base64.
+func EncryptURLString(key []byte, plaintext string) (string, error) {
+	cipherbytes, err := Encrypt(key, []byte(plaintext))
+	if err != nil {
+		return "", err
+	}
+	// convert to URL encoded base64
+	return base64.URLEncoding.EncodeToString(cipherbytes), nil
+}
+
+// EncryptString encrypts a string using AES and returns it in base64.
+func EncryptString(key []byte, plaintext string) (string, error) {
+	cipherbytes, err := Encrypt(key, []byte(plaintext))
+	if err != nil {
+		return "", err
+	}
+	// convert to (standard) encoded base64
+	return base64.StdEncoding.EncodeToString(cipherbytes), nil
+}
+
+// Decrypt an AES encrypted array of bytes.
+func Decrypt(key, ciphertext []byte) ([]byte, error) {
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, err
+	}
+
+	// The IV needs to be unique, but not secure. Therefore it's common to
+	// include it at the beginning of the ciphertext.
+	if len(ciphertext) < aes.BlockSize {
+		return nil, err
+	}
+	iv := ciphertext[:aes.BlockSize]
+	ciphertext = ciphertext[aes.BlockSize:]
+
+	stream := cipher.NewCFBDecrypter(block, iv)
+
+	// XORKeyStream can work in-place if the two arguments are the same.
+	stream.XORKeyStream(ciphertext, ciphertext)
+
+	return ciphertext, nil
+}
+
+// DecryptURLString decrypts an AES encrypted URL encoded base64 string.
+func DecryptURLString(key []byte, encstring string) (string, error) {
+	ciphertext, err := base64.URLEncoding.DecodeString(encstring)
+	if err != nil {
+		return "", err
+	}
+	plainbytes, err := Decrypt(key, ciphertext)
+	if err != nil {
+		return "", err
+	}
+	return string(plainbytes), nil
+}
+
+// DecryptString decrypts an AES encrypted base64 string.
+func DecryptString(key []byte, encstring string) (string, error) {
+	ciphertext, _ := base64.StdEncoding.DecodeString(encstring)
+	plainbytes, err := Decrypt(key, ciphertext)
+	if err != nil {
+		return "", err
+	}
+	return string(plainbytes), nil
+}

vendor/github.com/G-Node/libgin/libgin/dex.go

@@ -9,11 +9,9 @@ import (
 // NOTE: TEMPORARY COPY FROM gin-dex
 
 type SearchRequest struct {
-	Token  string
-	CsrfT  string
-	UserID int64
-	Query  string
-	SType  int64
+	Keywords string
+	SType    int
+	RepoIDs  []int64
 }
 
 const (
@@ -60,3 +58,9 @@ type IndexCommit struct {
 	GinRepoName  string
 	IndexingTime time.Time
 }
+
+type IndexRequest struct {
+	UserID   int64
+	RepoID   int64
+	RepoPath string
+}

vendor/github.com/G-Node/libgin/libgin/util.go

@@ -0,0 +1,24 @@
+package libgin
+
+// Common utilities for the GIN services
+
+import (
+	"os"
+)
+
+// ReadConfDefault returns the value of a configuration env variable.
+// If the variable is not set, the default is returned.
+func ReadConfDefault(key, defval string) string {
+	value, ok := os.LookupEnv(key)
+	if !ok {
+		return defval
+	}
+	return value
+}
+
+// ReadConf returns the value of a configuration env variable.
+// If the variable is not set, an empty string is returned (ignores any errors).
+func ReadConf(key string) string {
+	value, _ := os.LookupEnv(key)
+	return value
+}

vendor/vendor.json

@@ -50,10 +50,10 @@
 			"revision": ""
 		},
 		{
-			"checksumSHA1": "HM8GI780IYgrBjmQfEB8YXN4Gbs=",
+			"checksumSHA1": "Tt6eNIWKZnMwKILvKrvyf8Fsh7w=",
 			"path": "github.com/G-Node/libgin/libgin",
-			"revision": "c5e0b86f5b8b481b061df2926eea96e7e7793eca",
-			"revisionTime": "2018-12-13T18:01:10Z"
+			"revision": "c461e1ae688b84cf98a5f9d4fef087d0e15de750",
+			"revisionTime": "2019-07-01T17:53:26Z"
 		},
 		{
 			"checksumSHA1": "s+CthU3jnDZx1F6sOfsEvRLFjBE=",