db_gin.go 7.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276
  1. package db
  2. import (
  3. "bufio"
  4. "crypto/sha1"
  5. "encoding/hex"
  6. "encoding/json"
  7. "fmt"
  8. "io"
  9. "net/http"
  10. "os"
  11. "path"
  12. "path/filepath"
  13. "regexp"
  14. "strings"
  15. "github.com/G-Node/git-module"
  16. "github.com/G-Node/gogs/internal/setting"
  17. "github.com/G-Node/libgin/libgin"
  18. "github.com/G-Node/libgin/libgin/annex"
  19. "github.com/unknwon/com"
  20. "golang.org/x/crypto/bcrypt"
  21. log "gopkg.in/clog.v1"
  22. )
  23. // StartIndexing sends an indexing request to the configured indexing service
  24. // for a repository.
  25. func StartIndexing(repo Repository) {
  26. go func() {
  27. if setting.Search.IndexURL == "" {
  28. log.Trace("Indexing not enabled")
  29. return
  30. }
  31. log.Trace("Indexing repository %d", repo.ID)
  32. ireq := libgin.IndexRequest{
  33. RepoID: repo.ID,
  34. RepoPath: repo.FullName(),
  35. }
  36. data, err := json.Marshal(ireq)
  37. if err != nil {
  38. log.Error(2, "Could not marshal index request: %v", err)
  39. return
  40. }
  41. key := []byte(setting.Search.Key)
  42. encdata, err := libgin.EncryptString(key, string(data))
  43. if err != nil {
  44. log.Error(2, "Could not encrypt index request: %v", err)
  45. }
  46. req, err := http.NewRequest(http.MethodPost, setting.Search.IndexURL, strings.NewReader(encdata))
  47. if err != nil {
  48. log.Error(2, "Error creating index request")
  49. }
  50. client := http.Client{}
  51. resp, err := client.Do(req)
  52. if err != nil || resp.StatusCode != http.StatusOK {
  53. log.Error(2, "Error submitting index request for [%d: %s]: %v", repo.ID, repo.FullName(), err)
  54. return
  55. }
  56. }()
  57. }
  58. // RebuildIndex sends all repositories to the indexing service to be indexed.
  59. func RebuildIndex() error {
  60. indexurl := setting.Search.IndexURL
  61. if indexurl == "" {
  62. return fmt.Errorf("Indexing service not configured")
  63. }
  64. // collect all repo ID -> Path mappings directly from the DB
  65. repos := make(RepositoryList, 0, 100)
  66. if err := x.Find(&repos); err != nil {
  67. return fmt.Errorf("get all repos: %v", err)
  68. }
  69. log.Trace("Found %d repositories to index", len(repos))
  70. for _, repo := range repos {
  71. StartIndexing(*repo)
  72. }
  73. log.Trace("Rebuilding search index")
  74. return nil
  75. }
  76. func annexUninit(path string) {
  77. // walker sets the permission for any file found to 0660, to allow deletion
  78. var mode os.FileMode
  79. walker := func(path string, info os.FileInfo, err error) error {
  80. if info == nil {
  81. return nil
  82. }
  83. mode = 0660
  84. if info.IsDir() {
  85. mode = 0770
  86. }
  87. if err := os.Chmod(path, mode); err != nil {
  88. log.Error(3, "failed to change permissions on '%s': %v", path, err)
  89. }
  90. return nil
  91. }
  92. log.Trace("Uninit annex at '%s'", path)
  93. if msg, err := annex.Uninit(path); err != nil {
  94. log.Error(3, "uninit failed: %v (%s)", err, msg)
  95. if werr := filepath.Walk(path, walker); werr != nil {
  96. log.Error(3, "file permission change failed: %v", werr)
  97. }
  98. }
  99. }
  100. func annexSetup(path string) {
  101. log.Trace("Running annex add (with filesize filter) in '%s'", path)
  102. // Initialise annex in case it's a new repository
  103. if msg, err := annex.Init(path); err != nil {
  104. log.Error(2, "Annex init failed: %v (%s)", err, msg)
  105. return
  106. }
  107. // Upgrade to v8 in case the directory was here before and wasn't cleaned up properly
  108. if msg, err := annex.Upgrade(path); err != nil {
  109. log.Error(2, "Annex upgrade failed: %v (%s)", err, msg)
  110. return
  111. }
  112. // Enable addunlocked for annex v8
  113. if msg, err := annex.SetAddUnlocked(path); err != nil {
  114. log.Error(2, "Failed to set 'addunlocked' annex option: %v (%s)", err, msg)
  115. }
  116. // Set MD5 as default backend
  117. if msg, err := annex.MD5(path); err != nil {
  118. log.Error(2, "Failed to set default backend to 'MD5': %v (%s)", err, msg)
  119. }
  120. // Set size filter in config
  121. if msg, err := annex.SetAnnexSizeFilter(path, setting.Repository.Upload.AnnexFileMinSize*annex.MEGABYTE); err != nil {
  122. log.Error(2, "Failed to set size filter for annex: %v (%s)", err, msg)
  123. }
  124. }
  125. func annexSync(path string) error {
  126. log.Trace("Synchronising annexed data")
  127. if msg, err := annex.ASync(path, "--content"); err != nil {
  128. // TODO: This will also DOWNLOAD content, which is unnecessary for a simple upload
  129. // TODO: Use gin-cli upload function instead
  130. log.Error(2, "Annex sync failed: %v (%s)", err, msg)
  131. return fmt.Errorf("git annex sync --content [%s]", path)
  132. }
  133. // run twice; required if remote annex is not initialised
  134. if msg, err := annex.ASync(path, "--content"); err != nil {
  135. log.Error(2, "Annex sync failed: %v (%s)", err, msg)
  136. return fmt.Errorf("git annex sync --content [%s]", path)
  137. }
  138. return nil
  139. }
  140. func annexAdd(repoPath string, all bool, files ...string) error {
  141. cmd := git.NewCommand("annex", "add")
  142. if all {
  143. cmd.AddArguments(".")
  144. }
  145. _, err := cmd.AddArguments(files...).RunInDir(repoPath)
  146. return err
  147. }
  148. func annexUpload(repoPath, remote string) error {
  149. log.Trace("Synchronising annex info")
  150. if msg, err := git.NewCommand("annex", "sync").RunInDir(repoPath); err != nil {
  151. log.Error(2, "git-annex sync failed: %v (%s)", err, msg)
  152. }
  153. log.Trace("Uploading annexed data")
  154. cmd := git.NewCommand("annex", "copy", fmt.Sprintf("--to=%s", remote), "--all")
  155. if msg, err := cmd.RunInDir(repoPath); err != nil {
  156. log.Error(2, "git-annex copy failed: %v (%s)", err, msg)
  157. return fmt.Errorf("git annex copy [%s]", repoPath)
  158. }
  159. return nil
  160. }
  161. // isAddressAllowed returns true if the email address is allowed to sign up
  162. // based on the regular expressions found in the email filter file
  163. // (custom/addressfilters).
  164. // In case of errors (opening or reading file) or no matches, the function
  165. // defaults to 'true'.
  166. func isAddressAllowed(email string) bool {
  167. fpath := path.Join(setting.CustomPath, "addressfilters")
  168. if !com.IsExist(fpath) {
  169. // file doesn't exist: default allow everything
  170. return true
  171. }
  172. f, err := os.Open(fpath)
  173. if err != nil {
  174. log.Error(2, "Failed to open file %q: %v", fpath, err)
  175. // file read error: default allow everything
  176. return true
  177. }
  178. defer f.Close()
  179. emailBytes := []byte(email)
  180. scanner := bufio.NewScanner(f)
  181. for scanner.Scan() {
  182. // Check provided email address against each line regex
  183. // Failure to match any line returns true (allowed)
  184. // Matching a line prefixed with + returns true (allowed)
  185. // Matching a line prefixed with - returns false (blocked)
  186. // Erroneous patterns are logged and ignored
  187. var allow bool
  188. line := scanner.Text()
  189. if line[0] == '-' {
  190. allow = false
  191. } else if line[0] == '+' {
  192. allow = true
  193. } else {
  194. log.Error(2, "Invalid line in addressfilters: %s", line)
  195. log.Error(2, "Prefix invalid (must be '-' or '+')")
  196. continue
  197. }
  198. pattern := strings.TrimSpace(line[1:])
  199. match, err := regexp.Match(pattern, emailBytes)
  200. if err != nil {
  201. log.Error(2, "Invalid line in addressfilters: %s", line)
  202. log.Error(2, "Invalid pattern: %v", err)
  203. }
  204. if match {
  205. log.Trace("New user email %q matched filter rule %q (Allow: %t)", email, line, allow)
  206. return allow
  207. }
  208. }
  209. // No match: Default to allow
  210. return true
  211. }
  212. // isOnWhitelist returns true if the hash of a provided email address
  213. // can be found in an external whitelist file.
  214. // Returns false if the email address hash is not found or the file cannot be accessed.
  215. func isOnWhitelist(email string) bool {
  216. // BC20 whitelist location
  217. const whitelistlocation = "https://bc20-posters.g-node.org/uploads/emailwhitelist"
  218. // Hash email address
  219. hasher := sha1.New()
  220. io.WriteString(hasher, email)
  221. hash := hasher.Sum(nil)
  222. compare := hex.EncodeToString(hash[:])
  223. // Fetch whitelist
  224. resp, err := http.Get(whitelistlocation)
  225. if err != nil {
  226. log.Error(2, "Error fetching whitelist: '%s'", err.Error())
  227. return false
  228. }
  229. defer resp.Body.Close()
  230. // Check if provided email address is in whitelist
  231. var registered bool
  232. respScan := bufio.NewScanner(resp.Body)
  233. for respScan.Scan() {
  234. curr := respScan.Text()
  235. if curr == "" {
  236. continue
  237. }
  238. if curr == compare {
  239. registered = true
  240. break
  241. }
  242. }
  243. return registered
  244. }
  245. func (u *User) OldGinVerifyPassword(plain string) bool {
  246. err := bcrypt.CompareHashAndPassword([]byte(u.Passwd), []byte(plain))
  247. return err == nil
  248. }