From 7ebbbe5c29c5ecff9f15d6627084aa1ad9a889b8 Mon Sep 17 00:00:00 2001 From: Nicola Murino Date: Wed, 22 Jan 2020 19:42:23 +0100 Subject: [PATCH] S3: update pipeat to the latest commit in my fork Here are the main improvements: - unliked files works on windows too - the uploads are now synced on the lower speed between the SFTP client write and the upload speed to S3 This commit increase the external auth timeout to 60 seconds too --- README.md | 3 ++- dataprovider/dataprovider.go | 2 +- go.mod | 2 +- go.sum | 7 ++----- httpd/api_quota.go | 4 ++-- 5 files changed, 8 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 71a60704..4d828065 100644 --- a/README.md +++ b/README.md @@ -327,7 +327,7 @@ Previous global environment variables aren't cleared when the script is called. The program must respond on the standard output with a valid SFTPGo user serialized as JSON if the authentication succeed or an user with an empty username if the authentication fails. If the authentication succeed the user will be automatically added/updated inside the defined data provider. Actions defined for user added/updated will not be executed in this case. The external program should check authentication only, if there are login restrictions such as user disabled, expired, login allowed only from specific IP addresses it is enough to populate the matching user fields and these conditions will be checked in the same way as for built-in users. -The external auth program must finish within 15 seconds. +The external auth program should finish very quickly, anyway it will be killed if it does not exit within 60 seconds. This method is slower than built-in authentication, but it's very flexible as anyone can easily write his own authentication program. You can also restrict the authentication scope for the external program using the `external_auth_scope` configuration key: @@ -375,6 +375,7 @@ The program must write the questions on its standard output, in a single line, u - `auth_result`, integer. Set this field to 1 to indicate successfull authentication, 0 is ignored, any other value means authentication error. If this fields is found and it is different from 0 then SFTPGo does not read any other questions from the external program and finalize the authentication. SFTPGo writes the user answers to the program standard input, one per line, in the same order of the questions. +Please be sure that your program receive the answers for all the issued questions before asking for the next ones. Keyboard interactive authentication can be chained to the external authentication. The authentication must finish within 60 seconds. diff --git a/dataprovider/dataprovider.go b/dataprovider/dataprovider.go index 4c72d394..41d39cc4 100644 --- a/dataprovider/dataprovider.go +++ b/dataprovider/dataprovider.go @@ -847,7 +847,7 @@ func doKeyboardInteractiveAuth(user User, authProgram string, client ssh.Keyboar func doExternalAuth(username, password, pubKey, keyboardInteractive string) (User, error) { var user User - ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() pkey := "" if len(pubKey) > 0 { diff --git a/go.mod b/go.mod index 328d03cc..9b2d5bc8 100644 --- a/go.mod +++ b/go.mod @@ -27,4 +27,4 @@ require ( gopkg.in/natefinch/lumberjack.v2 v2.0.0 ) -replace github.com/eikenb/pipeat v0.0.0-20190316224601-fb1f3a9aa29f => github.com/drakkan/pipeat v0.0.0-20200114135659-fac71c64d75d +replace github.com/eikenb/pipeat v0.0.0-20190316224601-fb1f3a9aa29f => github.com/drakkan/pipeat v0.0.0-20200122173221-ea03f92ba172 diff --git a/go.sum b/go.sum index c213057d..65a06d2c 100644 --- a/go.sum +++ b/go.sum @@ -29,12 +29,11 @@ github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/drakkan/pipeat v0.0.0-20200114135659-fac71c64d75d h1:+k0oy9bBY9dXlKHriYg6crXpwIrtM1rCrlUehmc/F3M= -github.com/drakkan/pipeat v0.0.0-20200114135659-fac71c64d75d/go.mod h1:wNYvIpR5rIhoezOYcpxcXz4HbIEOu7A45EqlQCA+h+w= +github.com/drakkan/pipeat v0.0.0-20200122173221-ea03f92ba172 h1:0mSDVf/0IPuuy3c5Qg+ceoxgbV7KUoLN3Ircswglf7A= +github.com/drakkan/pipeat v0.0.0-20200122173221-ea03f92ba172/go.mod h1:wNYvIpR5rIhoezOYcpxcXz4HbIEOu7A45EqlQCA+h+w= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -117,7 +116,6 @@ github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.11.0 h1:4Zv0OGbpkg4yNuUtH0s8rvoYxRCNyT29NVUo6pgPmxI= github.com/pkg/sftp v1.11.0/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= @@ -168,7 +166,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= diff --git a/httpd/api_quota.go b/httpd/api_quota.go index 35df8ec2..6204c6f8 100644 --- a/httpd/api_quota.go +++ b/httpd/api_quota.go @@ -42,10 +42,10 @@ func doQuotaScan(user dataprovider.User) error { } numFiles, size, err := fs.ScanRootDirContents() if err != nil { - logger.Warn(logSender, "", "error scanning user home dir %#v: %v", user.HomeDir, err) + logger.Warn(logSender, "", "error scanning user home dir %#v: %v", user.Username, err) } else { err = dataprovider.UpdateUserQuota(dataProvider, user, numFiles, size, true) - logger.Debug(logSender, "", "user home dir scanned, user: %#v, dir: %#v, error: %v", user.Username, user.HomeDir, err) + logger.Debug(logSender, "", "user home dir scanned, user: %#v, error: %v", user.Username, err) } return err }