[models] Minimise upstream file diff

Split out code for adding annexed files during web upload, as well as
the annex sync and uninit operations.

This makes the differences between our fork's version of
models/repo_editor.go and the upstream much smaller.

models/admin_gin.go

@@ -1,20 +1,14 @@
 package models
 
 import (
-	"strings"
-
 	gannex "github.com/G-Node/go-annex"
 	log "gopkg.in/clog.v1"
 )
 
 func annexUninit(path string) {
+	log.Trace("Uninit annex at '%s'", path)
 	if msg, err := gannex.AUInit(path); err != nil {
-		if strings.Contains(msg, "If you don't care about preserving the data") {
-			log.Trace("Annex uninit Repo: %s", msg)
-		} else {
-			log.Error(1, "Could not annex uninit repo. Error: %s,%s", err, msg)
-		}
-	} else {
-		log.Trace("Annex uninit Repo:%s", msg)
+		log.Error(1, "uninit failed: %v (%s)", err, msg)
+		// TODO: Set mode 777 on all files to allow removal
 	}
 }

models/repo_editor.go

@@ -499,34 +499,9 @@ func (repo *Repository) UploadRepoFiles(doer *User, opts UploadRepoFileOptions)
 		if err = com.Copy(tmpPath, targetPath); err != nil {
 			return fmt.Errorf("copy: %v", err)
 		}
-		repoFileName := path.Join(opts.TreePath, upload.Name)
-
-		// needed for annex, due to symlinks
-		os.Remove(targetPath)
-		if err = com.Copy(tmpPath, targetPath); err != nil {
-			return fmt.Errorf("copy: %v", err)
-		}
-		log.Trace("Check for annexing: %s", upload.Name)
-		if finfo, err := os.Stat(targetPath); err == nil {
-			log.Trace("Filesize is:%d", finfo.Size())
-			// Should we annex
-			if finfo.Size() > setting.Repository.Upload.AnexFileMinSize*gannex.MEGABYTE {
-				log.Trace("This file should be annexed: %s", upload.Name)
-				// annex init in case it isnt yet
-				if msg, err := gannex.AInit(localPath, "annex.backend"); err != nil {
-					log.Error(1, "Annex init failed with error: %v,%s,%s", err, msg, repoFileName)
-				}
-				// worm for compatibility
-				gannex.Md5(localPath)
-				if msg, err := gannex.Add(repoFileName, localPath); err != nil {
-					log.Error(1, "Annex add failed with error: %v,%s,%s", err, msg, repoFileName)
-				}
-			}
-		} else {
-			log.Error(1, "could not stat: %s", targetPath)
-		}
 	}
 
+	annexAdd(localPath) // Running annex add with filter for big files
 	if err = git.AddChanges(localPath, true); err != nil {
 		return fmt.Errorf("git add --all: %v", err)
 	} else if err = git.CommitChanges(localPath, git.CommitChangesOptions{
@@ -546,27 +521,20 @@ func (repo *Repository) UploadRepoFiles(doer *User, opts UploadRepoFileOptions)
 		return fmt.Errorf("git push origin %s: %v", opts.NewBranch, err)
 	}
 
-	// Sometimes you need this twice
-	if msg, err := gannex.ASync(localPath, "--content", "--no-pull"); err != nil {
-		log.Error(1, "Annex sync failed with error: %v,%s", err, msg)
-	} else {
-		log.Trace("Annex sync:%s", msg)
-	}
-	if msg, err := gannex.ASync(localPath, "--content", "--no-pull"); err != nil {
-		log.Error(1, "Annex sync failed with error: %v,%s", err, msg)
-	} else {
-		log.Trace("Annex sync:%s", msg)
+	// GIN START
+	log.Trace("Synchronising annexed data")
+	if msg, err := gannex.ASync(localPath, "--content"); err != nil {
+		// TODO: This will also DOWNLOAD content, which is unnecessary for a simple upload
+		// TODO: Use gin-cli upload function instead
+		log.Error(1, "Annex sync failed: %v (%s)", err, msg)
 	}
 
-	// We better start out cleaning now. No use keeping files around with annex
-	if msg, err := gannex.AUInit(localPath); err != nil {
-		log.Error(1, "Annex uninit failed with error: %v,%s, at: %s/ This repository might fail at "+
-			"subsequent uploads!", err, msg, localPath)
-	}
+	annexUninit(localPath)
 	// Indexing support
 	if setting.Search.Do {
 		StartIndexing(doer, repo.MustOwner(), repo)
 	}
 	RemoveAllWithNotice("Cleaning out after upload", localPath)
+	// GIN END
 	return DeleteUploads(uploads...)
 }

models/repo_editor_gin.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"net/http"
 
+	gannex "github.com/G-Node/go-annex"
 	"github.com/G-Node/gogs/pkg/setting"
 	log "gopkg.in/clog.v1"
 )
@@ -29,3 +30,28 @@ func StartIndexing(user, owner *User, repo *Repository) {
 		return
 	}
 }
+
+func annexAdd(path string) {
+	log.Trace("Running annex add (with filesize filter) in '%s'", path)
+
+	// Initialise annex in case it's a new repository
+	if msg, err := gannex.AInit(path, "--version=7"); err != nil {
+		log.Error(1, "Annex init failed: %v (%s)", err, msg)
+		return
+	}
+
+	// Enable addunlocked for annex v7
+	if msg, err := gannex.SetAddUnlocked(path); err != nil {
+		log.Error(1, "Failed to set 'addunlocked' annex option: %v (%s)", err, msg)
+	}
+
+	// Set MD5 as default backend
+	if msg, err := gannex.Md5(path); err != nil {
+		log.Error(1, "Failed to set default backend to 'MD5': %v (%s)", err, msg)
+	}
+
+	sizefilterflag := fmt.Sprintf("--largerthan=%d", setting.Repository.Upload.AnexFileMinSize*gannex.MEGABYTE)
+	if msg, err := gannex.Add(path, sizefilterflag); err != nil {
+		log.Error(1, "Annex add failed with error: %v (%s)", err, msg)
+	}
+}