Merge pull request #62 from achilleas-k/annex-over-http

LGTM

internal/cmd/web.go

@@ -654,6 +654,13 @@ func runWeb(c *cli.Context) error {
 		m.Get("/watchers", repo.Watchers)
 	}, ignSignIn, context.RepoAssignment(), context.RepoRef())
 
+	m.Group("/:username/:reponame", func() {
+		// GIN mod: Annex over HTTP
+		m.Get("/config", repo.GitConfig)
+		m.Get("/annex/objects/:hashdira/:hashdirb/:key/:keyfile", repo.AnnexGetKey)
+		m.Head("/annex/objects/:hashdira/:hashdirb/:key/:keyfile", repo.AnnexGetKey)
+	}, ignSignInAndCsrf, context.RepoAssignment())
+
 	m.Group("/:username", func() {
 		m.Get("/:reponame", ignSignIn, context.RepoAssignment(), context.RepoRef(), repo.Home)
 
@@ -666,10 +673,10 @@ func runWeb(c *cli.Context) error {
 		m.Group("/:reponame([\\d\\w-_\\.]+\\.git$)", func() {
 			m.Get("", ignSignIn, context.RepoAssignment(), context.RepoRef(), repo.Home)
 			m.Options("/*", ignSignInAndCsrf, repo.HTTPContexter(), repo.HTTP)
-			m.Route("/*", "GET,POST,HEAD", ignSignInAndCsrf, repo.HTTPContexter(), repo.HTTP)
+			m.Route("/*", "GET,POST", ignSignInAndCsrf, repo.HTTPContexter(), repo.HTTP)
 		})
 		m.Options("/:reponame/*", ignSignInAndCsrf, repo.HTTPContexter(), repo.HTTP)
-		m.Route("/:reponame/*", "GET,POST,HEAD", ignSignInAndCsrf, repo.HTTPContexter(), repo.HTTP)
+		m.Route("/:reponame/*", "GET,POST", ignSignInAndCsrf, repo.HTTPContexter(), repo.HTTP)
 	})
 	// ***** END: Repository *****
 

internal/route/repo/http.go

@@ -62,7 +62,7 @@ func HTTPContexter() macaron.Handler {
 
 		isPull := c.Query("service") == "git-upload-pack" ||
 			strings.HasSuffix(c.Req.URL.Path, "git-upload-pack") ||
-			c.Req.Method == "GET" || c.Req.Method == "HEAD"
+			c.Req.Method == "GET"
 
 		owner, err := db.GetUserByName(ownerName)
 		if err != nil {
@@ -325,11 +325,6 @@ func getTextFile(h serviceHandler) {
 	h.sendFile("text/plain")
 }
 
-func getBinaryFile(h serviceHandler) {
-	h.setHeaderNoCache()
-	h.sendFile("application/octet-stream")
-}
-
 func getInfoPacks(h serviceHandler) {
 	h.setHeaderCacheForever()
 	h.sendFile("text/plain; charset=utf-8")
@@ -366,12 +361,6 @@ var routes = []struct {
 	{regexp.MustCompile("(.*?)/objects/[0-9a-f]{2}/[0-9a-f]{38}$"), "GET", getLooseObject},
 	{regexp.MustCompile("(.*?)/objects/pack/pack-[0-9a-f]{40}\\.pack$"), "GET", getPackFile},
 	{regexp.MustCompile("(.*?)/objects/pack/pack-[0-9a-f]{40}\\.idx$"), "GET", getIdxFile},
-	// files neeeded for git-annex access to the repository over http
-	{regexp.MustCompile("(.*?)/config$"), "GET", getTextFile},
-	// TODO: we probably just need to provide some getBinaryFile
-	// Note: code below treats HEAD (used by git annex drop to sense presence)
-	// as "GET" for the purpose of allowing or not the route
-	{regexp.MustCompile("(.*?)/annex/objects/.*/.*-s[0-9]*--.*"), "GET", getBinaryFile},
 }
 
 func getGitRepoPath(dir string) (string, error) {
@@ -388,14 +377,8 @@ func getGitRepoPath(dir string) (string, error) {
 }
 
 func HTTP(c *HTTPContext) {
-    var reqPath string
 	for _, route := range routes {
-		// Annex keys are case sensitive, so they must not be lower cased
-		if strings.Contains(c.Req.URL.Path, "/annex/objects/") {
-			reqPath = c.Req.URL.Path
-		} else {
-			reqPath = strings.ToLower(c.Req.URL.Path)
-		}
+		reqPath := strings.ToLower(c.Req.URL.Path)
 		m := route.reg.FindStringSubmatch(reqPath)
 		if m == nil {
 			continue
@@ -409,10 +392,7 @@ func HTTP(c *HTTPContext) {
 			return
 		}
 
-		req_method := c.Req.Method
-		// Treat HEAD (used by e.g. git annex drop) as GET
-		if req_method == "HEAD" { req_method = "GET" }
-		if route.method != req_method {
+		if route.method != c.Req.Method {
 			c.NotFound()
 			return
 		}

internal/route/repo/repo_gin.go

@@ -3,39 +3,57 @@ package repo
 import (
 	"bufio"
 	"bytes"
+	"fmt"
 	"io"
 	"io/ioutil"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
 
 	"github.com/G-Node/git-module"
 	"github.com/G-Node/gogs/internal/context"
 	"github.com/G-Node/gogs/internal/setting"
 	"github.com/G-Node/gogs/internal/tool"
 	"github.com/G-Node/libgin/libgin"
-	"github.com/G-Node/libgin/libgin/annex"
 	"github.com/go-macaron/captcha"
 	log "gopkg.in/clog.v1"
 	"gopkg.in/yaml.v2"
 )
 
 func serveAnnexedData(ctx *context.Context, name string, cpt *captcha.Captcha, buf []byte) error {
-	annexFile, err := annex.NewAFile(ctx.Repo.Repository.RepoPath(), "annex", name, buf)
+	keyparts := strings.Split(strings.TrimSpace(string(buf)), "/")
+	key := keyparts[len(keyparts)-1]
+	contentPath, err := git.NewCommand("annex", "contentlocation", key).RunInDir(ctx.Repo.Repository.RepoPath())
 	if err != nil {
+		log.Error(2, "Failed to find content location for file %q with key %q", name, key)
 		return err
 	}
-	if cpt != nil && annexFile.Info.Size() > annex.MEGABYTE*setting.Repository.RawCaptchaMinFileSize && !cpt.VerifyReq(ctx.Req) &&
-		!ctx.IsLogged {
-		ctx.Data["EnableCaptcha"] = true
-		ctx.HTML(200, "repo/download")
-		return nil
-	}
-	annexfp, err := annexFile.Open()
+	// always trim space from output for git command
+	contentPath = strings.TrimSpace(contentPath)
+	return serveAnnexedKey(ctx, name, contentPath)
+}
+
+func serveAnnexedKey(ctx *context.Context, name string, contentPath string) error {
+	fullContentPath := filepath.Join(ctx.Repo.Repository.RepoPath(), contentPath)
+	annexfp, err := os.Open(fullContentPath)
 	if err != nil {
+		log.Error(2, "Failed to open annex file at %q: %s", fullContentPath, err.Error())
 		return err
 	}
 	defer annexfp.Close()
 	annexReader := bufio.NewReader(annexfp)
-	buf, _ = annexReader.Peek(1024)
 
+	info, err := annexfp.Stat()
+	if err != nil {
+		log.Error(2, "Failed to stat file at %q: %s", fullContentPath, err.Error())
+		return err
+	}
+
+	buf, _ := annexReader.Peek(1024)
+
+	ctx.Resp.Header().Set("Content-Length", fmt.Sprintf("%d", info.Size()))
 	if !tool.IsTextFile(buf) {
 		if !tool.IsImageFile(buf) {
 			ctx.Resp.Header().Set("Content-Disposition", "attachment; filename=\""+name+"\"")
@@ -45,6 +63,12 @@ func serveAnnexedData(ctx *context.Context, name string, cpt *captcha.Captcha, b
 		ctx.Resp.Header().Set("Content-Type", "text/plain; charset=utf-8")
 	}
 
+	log.Trace("Serving annex content for %q: %q", name, contentPath)
+	if ctx.Req.Method == http.MethodHead {
+		// Skip content copy when request method is HEAD
+		log.Trace("Returning header: %+v", ctx.Resp.Header())
+		return nil
+	}
 	_, err = io.Copy(ctx.Resp, annexReader)
 	return err
 }
@@ -103,25 +127,57 @@ func resolveAnnexedContent(c *context.Context, buf []byte, dataRc io.Reader) ([]
 		// not an annex pointer file; return as is
 		return buf, dataRc, nil
 	}
-	log.Trace("Annexed file requested: Resolving content for [%s]", bytes.TrimSpace(buf))
-	af, err := annex.NewAFile(c.Repo.Repository.RepoPath(), "annex", "", buf)
+	log.Trace("Annexed file requested: Resolving content for %q", bytes.TrimSpace(buf))
+
+	keyparts := strings.Split(strings.TrimSpace(string(buf)), "/")
+	key := keyparts[len(keyparts)-1]
+	contentPath, err := git.NewCommand("annex", "contentlocation", key).RunInDir(c.Repo.Repository.RepoPath())
 	if err != nil {
-		log.Trace("Could not get annex file: %v", err)
+		log.Error(2, "Failed to find content location for key %q", key)
 		c.Data["IsAnnexedFile"] = true
 		return buf, dataRc, err
 	}
-
-	afp, err := af.Open()
+	// always trim space from output for git command
+	contentPath = strings.TrimSpace(contentPath)
+	afp, err := os.Open(filepath.Join(c.Repo.Repository.RepoPath(), contentPath))
 	if err != nil {
 		log.Trace("Could not open annex file: %v", err)
 		c.Data["IsAnnexedFile"] = true
 		return buf, dataRc, err
 	}
+	info, err := afp.Stat()
+	if err != nil {
+		log.Trace("Could not stat annex file: %v", err)
+		c.Data["IsAnnexedFile"] = true
+		return buf, dataRc, err
+	}
 	annexDataReader := bufio.NewReader(afp)
 	annexBuf := make([]byte, 1024)
 	n, _ := annexDataReader.Read(annexBuf)
 	annexBuf = annexBuf[:n]
-	c.Data["FileSize"] = af.Info.Size()
-	log.Trace("Annexed file size: %d B", af.Info.Size())
+	c.Data["FileSize"] = info.Size()
+	log.Trace("Annexed file size: %d B", info.Size())
 	return annexBuf, annexDataReader, nil
 }
+
+func GitConfig(c *context.Context) {
+	log.Trace("RepoPath: %+v", c.Repo.Repository.RepoPath())
+	configFilePath := path.Join(c.Repo.Repository.RepoPath(), "config")
+	log.Trace("Serving file %q", configFilePath)
+	if _, err := os.Stat(configFilePath); err != nil {
+		c.ServerError("GitConfig", err)
+		return
+	}
+	c.ServeFileContent(configFilePath, "config")
+}
+
+func AnnexGetKey(c *context.Context) {
+	filename := c.Params(":keyfile")
+	key := c.Params(":key")
+	contentPath := filepath.Join("annex/objects", c.Params(":hashdira"), c.Params(":hashdirb"), key, filename)
+	log.Trace("Git annex requested key %q: %q", key, contentPath)
+	err := serveAnnexedKey(c, filename, contentPath)
+	if err != nil {
+		c.ServerError("AnnexGetKey", err)
+	}
+}

internal/route/repo/view.go

@@ -155,7 +155,8 @@ func renderFile(c *context.Context, entry *git.TreeEntry, treeLink, rawLink stri
 	n, _ := dataRc.Read(buf)
 	buf = buf[:n]
 
-	// GIN mod: Replace existing buf and reader with annexed content buf and reader
+	// GIN mod: Replace existing buf and reader with annexed content buf and
+	// reader (only if it's an annexed ptr file)
 	buf, dataRc, err = resolveAnnexedContent(c, buf, dataRc)
 	if err != nil {
 		return