diff --git a/config/config.go b/config/config.go index 776f8dda..29a0874a 100644 --- a/config/config.go +++ b/config/config.go @@ -251,6 +251,7 @@ func Init() { CACertificates: nil, CARevocationLists: nil, SigningPassphrase: "", + MaxUploadFileSize: 1048576000, }, HTTPConfig: httpclient.Config{ Timeout: 20, @@ -1039,6 +1040,7 @@ func setViperDefaults() { viper.SetDefault("httpd.ca_certificates", globalConf.HTTPDConfig.CACertificates) viper.SetDefault("httpd.ca_revocation_lists", globalConf.HTTPDConfig.CARevocationLists) viper.SetDefault("httpd.signing_passphrase", globalConf.HTTPDConfig.SigningPassphrase) + viper.SetDefault("httpd.max_upload_file_size", globalConf.HTTPDConfig.MaxUploadFileSize) viper.SetDefault("http.timeout", globalConf.HTTPConfig.Timeout) viper.SetDefault("http.retry_wait_min", globalConf.HTTPConfig.RetryWaitMin) viper.SetDefault("http.retry_wait_max", globalConf.HTTPConfig.RetryWaitMax) diff --git a/docs/full-configuration.md b/docs/full-configuration.md index 70224c85..ad6e786e 100644 --- a/docs/full-configuration.md +++ b/docs/full-configuration.md @@ -213,6 +213,7 @@ The configuration file contains the following sections: - `ca_certificates`, list of strings. Set of root certificate authorities to be used to verify client certificates. - `ca_revocation_lists`, list of strings. Set a revocation lists, one for each root CA, to be used to check if a client certificate has been revoked. The revocation lists can be reloaded on demand sending a `SIGHUP` signal on Unix based systems and a `paramchange` request to the running service on Windows. - `signing_passphrase`, string. Passphrase to use to derive the signing key for JWT and CSRF tokens. If empty a random signing key will be generated each time SFTPGo starts. If you set a signing passphrase you should consider rotating it periodically for added security. + - `max_upload_file_size`, integer. Defines the maximum request body size, in bytes, for Web Client/API HTTP upload requests. 0 means no limit. Default: 1048576000. - **"telemetry"**, the configuration for the telemetry server, more details [below](#telemetry-server) - `bind_port`, integer. The port used for serving HTTP requests. Set to 0 to disable HTTP server. Default: 10000 - `bind_address`, string. Leave blank to listen on all available network interfaces. On \*NIX you can specify an absolute path to listen on a Unix-domain socket. Default: "127.0.0.1" diff --git a/go.mod b/go.mod index 76643fd4..160aed9d 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962 github.com/StackExchange/wmi v1.2.0 // indirect github.com/alexedwards/argon2id v0.0.0-20210511081203-7d35d68092b8 - github.com/aws/aws-sdk-go v1.40.3 + github.com/aws/aws-sdk-go v1.40.6 github.com/cockroachdb/cockroach-go/v2 v2.1.1 github.com/eikenb/pipeat v0.0.0-20210603033007-44fc3ffce52b github.com/fatih/color v1.12.0 // indirect @@ -60,9 +60,9 @@ require ( golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a golang.org/x/net v0.0.0-20210614182718-04defd469f4e golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c - golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 - google.golang.org/api v0.50.0 - google.golang.org/genproto v0.0.0-20210719143636-1d5a45f8e492 // indirect + golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac + google.golang.org/api v0.51.0 + google.golang.org/genproto v0.0.0-20210722135532-667f2b7c528f // indirect google.golang.org/grpc v1.39.0 google.golang.org/protobuf v1.27.1 gopkg.in/natefinch/lumberjack.v2 v2.0.0 diff --git a/go.sum b/go.sum index fa4b6431..3d82dbe3 100644 --- a/go.sum +++ b/go.sum @@ -22,8 +22,9 @@ cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECH cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= -cloud.google.com/go v0.84.0 h1:hVhK90DwCdOAYGME/FJd9vNIZye9HBR6Yy3fu4js3N8= cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0 h1:8ZtzmY4a2JIO2sljMbpqkDYxA8aJQveYr3AMa+X40oc= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -121,8 +122,8 @@ github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZo github.com/aws/aws-sdk-go v1.23.20/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= -github.com/aws/aws-sdk-go v1.40.3 h1:NzjcLRsb+C9L1dVPajdNbdzkuPBi0pQJWiQW0eYJGo8= -github.com/aws/aws-sdk-go v1.40.3/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= +github.com/aws/aws-sdk-go v1.40.6 h1:JCQfi5MD8cW0PCAzr88hj9tj4BdEJkAy8EyAJ6c8I/k= +github.com/aws/aws-sdk-go v1.40.6/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -280,6 +281,7 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -338,6 +340,7 @@ github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= @@ -951,8 +954,8 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 h1:Vv0JUPWTyeqUq42B2WJ1FeIDjjvGKoA2Ss+Ts0lAVbs= -golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1059,8 +1062,9 @@ google.golang.org/api v0.46.0/go.mod h1:ceL4oozhkAiTID8XMmJBsIxID/9wMXJVVFXPg4yl google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= google.golang.org/api v0.49.0/go.mod h1:BECiH72wsfwUvOVn3+btPD5WHi0LzavZReBndi42L18= -google.golang.org/api v0.50.0 h1:LX7NFCFYOHzr7WHaYiRUpeipZe9o5L8T+2F4Z798VDw= google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0 h1:SQaA2Cx57B+iPw2MBgyjEkoeMkRK2IenSGoia0U3lCk= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1127,8 +1131,10 @@ google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxH google.golang.org/genproto v0.0.0-20210617175327-b9e0b3197ced/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= google.golang.org/genproto v0.0.0-20210624174822-c5cf32407d0a/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= -google.golang.org/genproto v0.0.0-20210719143636-1d5a45f8e492 h1:7yQQsvnwjfEahbNNEKcBHv3mR+HnB1ctGY/z1JXzx8M= -google.golang.org/genproto v0.0.0-20210719143636-1d5a45f8e492/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210722135532-667f2b7c528f h1:YORWxaStkWBnWgELOHTmDrqNlFXuVGEbhwbB5iK94bQ= +google.golang.org/genproto v0.0.0-20210722135532-667f2b7c528f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= diff --git a/httpd/api_http_user.go b/httpd/api_http_user.go index e0245ab8..4986c673 100644 --- a/httpd/api_http_user.go +++ b/httpd/api_http_user.go @@ -4,7 +4,10 @@ import ( "context" "errors" "fmt" + "io" "net/http" + "os" + "path" "time" "github.com/go-chi/render" @@ -12,30 +15,39 @@ import ( "github.com/drakkan/sftpgo/v2/common" "github.com/drakkan/sftpgo/v2/dataprovider" + "github.com/drakkan/sftpgo/v2/logger" "github.com/drakkan/sftpgo/v2/util" ) -func readUserFolder(w http.ResponseWriter, r *http.Request) { +func getUserConnection(w http.ResponseWriter, r *http.Request) (*Connection, error) { claims, err := getTokenClaims(r) if err != nil || claims.Username == "" { sendAPIResponse(w, r, err, "Invalid token claims", http.StatusBadRequest) - return + return nil, fmt.Errorf("invalid token claims %w", err) } user, err := dataprovider.UserExists(claims.Username) if err != nil { sendAPIResponse(w, r, nil, "Unable to retrieve your user", getRespStatus(err)) - return + return nil, err } connID := xid.New().String() connectionID := fmt.Sprintf("%v_%v", common.ProtocolHTTP, connID) if err := checkHTTPClientUser(&user, r, connectionID); err != nil { sendAPIResponse(w, r, err, http.StatusText(http.StatusForbidden), http.StatusForbidden) - return + return nil, err } connection := &Connection{ BaseConnection: common.NewBaseConnection(connID, common.ProtocolHTTP, r.RemoteAddr, user), request: r, } + return connection, nil +} + +func readUserFolder(w http.ResponseWriter, r *http.Request) { + connection, err := getUserConnection(w, r) + if err != nil { + return + } common.Connections.Add(connection) defer common.Connections.Remove(connection.GetID()) @@ -60,26 +72,66 @@ func readUserFolder(w http.ResponseWriter, r *http.Request) { render.JSON(w, r, results) } -func getUserFile(w http.ResponseWriter, r *http.Request) { - claims, err := getTokenClaims(r) - if err != nil || claims.Username == "" { - sendAPIResponse(w, r, err, "Invalid token claims", http.StatusBadRequest) - return - } - user, err := dataprovider.UserExists(claims.Username) +func createUserDir(w http.ResponseWriter, r *http.Request) { + r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize) + connection, err := getUserConnection(w, r) if err != nil { - sendAPIResponse(w, r, nil, "Unable to retrieve your user", getRespStatus(err)) return } - connID := xid.New().String() - connectionID := fmt.Sprintf("%v_%v", common.ProtocolHTTP, connID) - if err := checkHTTPClientUser(&user, r, connectionID); err != nil { - sendAPIResponse(w, r, err, http.StatusText(http.StatusForbidden), http.StatusForbidden) + common.Connections.Add(connection) + defer common.Connections.Remove(connection.GetID()) + + name := util.CleanPath(r.URL.Query().Get("path")) + err = connection.CreateDir(name) + if err != nil { + sendAPIResponse(w, r, err, fmt.Sprintf("Unable to create directory %#v", name), getMappedStatusCode(err)) return } - connection := &Connection{ - BaseConnection: common.NewBaseConnection(connID, common.ProtocolHTTP, r.RemoteAddr, user), - request: r, + sendAPIResponse(w, r, nil, fmt.Sprintf("Directory %#v created", name), http.StatusCreated) +} + +func renameUserDir(w http.ResponseWriter, r *http.Request) { + r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize) + connection, err := getUserConnection(w, r) + if err != nil { + return + } + common.Connections.Add(connection) + defer common.Connections.Remove(connection.GetID()) + + oldName := util.CleanPath(r.URL.Query().Get("path")) + newName := util.CleanPath(r.URL.Query().Get("target")) + err = connection.Rename(oldName, newName) + if err != nil { + sendAPIResponse(w, r, err, fmt.Sprintf("Unable to rename directory %#v to %#v", oldName, newName), + getMappedStatusCode(err)) + return + } + sendAPIResponse(w, r, nil, fmt.Sprintf("Directory %#v renamed to %#v", oldName, newName), http.StatusOK) +} + +func deleteUserDir(w http.ResponseWriter, r *http.Request) { + r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize) + connection, err := getUserConnection(w, r) + if err != nil { + return + } + common.Connections.Add(connection) + defer common.Connections.Remove(connection.GetID()) + + name := util.CleanPath(r.URL.Query().Get("path")) + err = connection.RemoveDir(name) + if err != nil { + sendAPIResponse(w, r, err, fmt.Sprintf("Unable to delete directory %#v", name), getMappedStatusCode(err)) + return + } + sendAPIResponse(w, r, nil, fmt.Sprintf("Directory %#v deleted", name), http.StatusOK) +} + +func getUserFile(w http.ResponseWriter, r *http.Request) { + connection, err := getUserConnection(w, r) + if err != nil { + return } common.Connections.Add(connection) defer common.Connections.Remove(connection.GetID()) @@ -112,26 +164,121 @@ func getUserFile(w http.ResponseWriter, r *http.Request) { } } -func getUserFilesAsZipStream(w http.ResponseWriter, r *http.Request) { - claims, err := getTokenClaims(r) - if err != nil || claims.Username == "" { - sendAPIResponse(w, r, err, "Invalid token claims", http.StatusBadRequest) - return +func uploadUserFiles(w http.ResponseWriter, r *http.Request) { + if maxUploadFileSize > 0 { + r.Body = http.MaxBytesReader(w, r.Body, maxUploadFileSize) } - user, err := dataprovider.UserExists(claims.Username) + + connection, err := getUserConnection(w, r) if err != nil { - sendAPIResponse(w, r, nil, "Unable to retrieve your user", getRespStatus(err)) return } - connID := xid.New().String() - connectionID := fmt.Sprintf("%v_%v", common.ProtocolHTTP, connID) - if err := checkHTTPClientUser(&user, r, connectionID); err != nil { - sendAPIResponse(w, r, err, http.StatusText(http.StatusForbidden), http.StatusForbidden) + common.Connections.Add(connection) + defer common.Connections.Remove(connection.GetID()) + + err = r.ParseMultipartForm(maxMultipartMem) + if err != nil { + sendAPIResponse(w, r, err, "Unable to parse multipart form", http.StatusBadRequest) return } - connection := &Connection{ - BaseConnection: common.NewBaseConnection(connID, common.ProtocolHTTP, r.RemoteAddr, user), - request: r, + + parentDir := util.CleanPath(r.URL.Query().Get("path")) + files := r.MultipartForm.File["filename"] + if len(files) == 0 { + sendAPIResponse(w, r, err, "No files uploaded!", http.StatusBadRequest) + return + } + + for _, f := range files { + file, err := f.Open() + if err != nil { + sendAPIResponse(w, r, err, fmt.Sprintf("Unable to read uploaded file %#v", f.Filename), getMappedStatusCode(err)) + return + } + defer file.Close() + + filePath := path.Join(parentDir, f.Filename) + writer, err := connection.getFileWriter(filePath) + if err != nil { + sendAPIResponse(w, r, err, fmt.Sprintf("Unable to write file %#v", f.Filename), getMappedStatusCode(err)) + return + } + _, err = io.Copy(writer, file) + if err != nil { + writer.Close() //nolint:errcheck + sendAPIResponse(w, r, err, fmt.Sprintf("Error saving file %#v", f.Filename), getMappedStatusCode(err)) + return + } + err = writer.Close() + if err != nil { + sendAPIResponse(w, r, err, fmt.Sprintf("Error closing file %#v", f.Filename), getMappedStatusCode(err)) + return + } + } + sendAPIResponse(w, r, nil, "Upload completed", http.StatusCreated) +} + +func renameUserFile(w http.ResponseWriter, r *http.Request) { + r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize) + connection, err := getUserConnection(w, r) + if err != nil { + return + } + common.Connections.Add(connection) + defer common.Connections.Remove(connection.GetID()) + + oldName := util.CleanPath(r.URL.Query().Get("path")) + newName := util.CleanPath(r.URL.Query().Get("target")) + err = connection.Rename(oldName, newName) + if err != nil { + sendAPIResponse(w, r, err, fmt.Sprintf("Unable to rename file %#v to %#v", oldName, newName), + getMappedStatusCode(err)) + return + } + sendAPIResponse(w, r, nil, fmt.Sprintf("File %#v renamed to %#v", oldName, newName), http.StatusOK) +} + +func deleteUserFile(w http.ResponseWriter, r *http.Request) { + r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize) + connection, err := getUserConnection(w, r) + if err != nil { + return + } + common.Connections.Add(connection) + defer common.Connections.Remove(connection.GetID()) + + name := util.CleanPath(r.URL.Query().Get("path")) + fs, p, err := connection.GetFsAndResolvedPath(name) + if err != nil { + sendAPIResponse(w, r, err, fmt.Sprintf("Unable to delete file %#v", name), getMappedStatusCode(err)) + return + } + + var fi os.FileInfo + if fi, err = fs.Lstat(p); err != nil { + connection.Log(logger.LevelWarn, "failed to remove a file %#v: stat error: %+v", p, err) + err = connection.GetFsError(fs, err) + sendAPIResponse(w, r, err, fmt.Sprintf("Unable to delete file %#v", name), getMappedStatusCode(err)) + return + } + + if fi.IsDir() && fi.Mode()&os.ModeSymlink == 0 { + connection.Log(logger.LevelDebug, "cannot remove %#v is not a file/symlink", p) + sendAPIResponse(w, r, err, fmt.Sprintf("Unable delete %#v, it is not a file/symlink", name), http.StatusBadRequest) + return + } + err = connection.RemoveFile(fs, p, name, fi) + if err != nil { + sendAPIResponse(w, r, err, fmt.Sprintf("Unable to delete file %#v", name), getMappedStatusCode(err)) + return + } + sendAPIResponse(w, r, nil, fmt.Sprintf("File %#v deleted", name), http.StatusOK) +} + +func getUserFilesAsZipStream(w http.ResponseWriter, r *http.Request) { + connection, err := getUserConnection(w, r) + if err != nil { + return } common.Connections.Add(connection) defer common.Connections.Remove(connection.GetID()) diff --git a/httpd/file.go b/httpd/file.go index 98aed8b7..63c98c05 100644 --- a/httpd/file.go +++ b/httpd/file.go @@ -8,25 +8,32 @@ import ( "github.com/eikenb/pipeat" "github.com/drakkan/sftpgo/v2/common" + "github.com/drakkan/sftpgo/v2/vfs" ) var errTransferAborted = errors.New("transfer aborted") type httpdFile struct { *common.BaseTransfer + writer io.WriteCloser reader io.ReadCloser isFinished bool } -func newHTTPDFile(baseTransfer *common.BaseTransfer, pipeReader *pipeat.PipeReaderAt) *httpdFile { +func newHTTPDFile(baseTransfer *common.BaseTransfer, pipeWriter *vfs.PipeWriter, pipeReader *pipeat.PipeReaderAt) *httpdFile { + var writer io.WriteCloser var reader io.ReadCloser if baseTransfer.File != nil { + writer = baseTransfer.File reader = baseTransfer.File + } else if pipeWriter != nil { + writer = pipeWriter } else if pipeReader != nil { reader = pipeReader } return &httpdFile{ BaseTransfer: baseTransfer, + writer: writer, reader: reader, isFinished: false, } @@ -51,6 +58,28 @@ func (f *httpdFile) Read(p []byte) (n int, err error) { return } +// Write writes the contents to upload +func (f *httpdFile) Write(p []byte) (n int, err error) { + if atomic.LoadInt32(&f.AbortTransfer) == 1 { + return 0, errTransferAborted + } + + f.Connection.UpdateLastActivity() + + n, err = f.writer.Write(p) + atomic.AddInt64(&f.BytesReceived, int64(n)) + + if f.MaxWriteSize > 0 && err == nil && atomic.LoadInt64(&f.BytesReceived) > f.MaxWriteSize { + err = common.ErrQuotaExceeded + } + if err != nil { + f.TransferError(err) + return + } + f.HandleThrottle() + return +} + // Close closes the current transfer func (f *httpdFile) Close() error { if err := f.setFinished(); err != nil { @@ -69,6 +98,14 @@ func (f *httpdFile) closeIO() error { var err error if f.File != nil { err = f.File.Close() + } else if f.writer != nil { + err = f.writer.Close() + f.Lock() + // we set ErrTransfer here so quota is not updated, in this case the uploads are atomic + if err != nil && f.ErrTransfer == nil { + f.ErrTransfer = err + } + f.Unlock() } else if f.reader != nil { err = f.reader.Close() } diff --git a/httpd/handler.go b/httpd/handler.go index be5bc561..9b7c0b32 100644 --- a/httpd/handler.go +++ b/httpd/handler.go @@ -11,6 +11,7 @@ import ( "github.com/drakkan/sftpgo/v2/dataprovider" "github.com/drakkan/sftpgo/v2/logger" "github.com/drakkan/sftpgo/v2/util" + "github.com/drakkan/sftpgo/v2/vfs" ) // Connection details for a HTTP connection used to inteact with an SFTPGo filesystem @@ -107,5 +108,104 @@ func (c *Connection) getFileReader(name string, offset int64, method string) (io baseTransfer := common.NewBaseTransfer(file, c.BaseConnection, cancelFn, p, p, name, common.TransferDownload, 0, 0, 0, false, fs) - return newHTTPDFile(baseTransfer, r), nil + return newHTTPDFile(baseTransfer, nil, r), nil +} + +func (c *Connection) getFileWriter(name string) (io.WriteCloser, error) { + c.UpdateLastActivity() + + if !c.User.IsFileAllowed(name) { + c.Log(logger.LevelWarn, "writing file %#v is not allowed", name) + return nil, c.GetPermissionDeniedError() + } + + fs, p, err := c.GetFsAndResolvedPath(name) + if err != nil { + return nil, err + } + filePath := p + if common.Config.IsAtomicUploadEnabled() && fs.IsAtomicUploadSupported() { + filePath = fs.GetAtomicUploadPath(p) + } + + stat, statErr := fs.Lstat(p) + if (statErr == nil && stat.Mode()&os.ModeSymlink != 0) || fs.IsNotExist(statErr) { + if !c.User.HasPerm(dataprovider.PermUpload, path.Dir(name)) { + return nil, c.GetPermissionDeniedError() + } + return c.handleUploadFile(fs, p, filePath, name, true, 0) + } + + if statErr != nil { + c.Log(logger.LevelError, "error performing file stat %#v: %+v", p, statErr) + return nil, c.GetFsError(fs, statErr) + } + + // This happen if we upload a file that has the same name of an existing directory + if stat.IsDir() { + c.Log(logger.LevelWarn, "attempted to open a directory for writing to: %#v", p) + return nil, c.GetOpUnsupportedError() + } + + if !c.User.HasPerm(dataprovider.PermOverwrite, path.Dir(name)) { + return nil, c.GetPermissionDeniedError() + } + + if common.Config.IsAtomicUploadEnabled() && fs.IsAtomicUploadSupported() { + err = fs.Rename(p, filePath) + if err != nil { + c.Log(logger.LevelWarn, "error renaming existing file for atomic upload, source: %#v, dest: %#v, err: %+v", + p, filePath, err) + return nil, c.GetFsError(fs, err) + } + } + + return c.handleUploadFile(fs, p, filePath, name, false, stat.Size()) +} + +func (c *Connection) handleUploadFile(fs vfs.Fs, resolvedPath, filePath, requestPath string, isNewFile bool, fileSize int64) (io.WriteCloser, error) { + quotaResult := c.HasSpace(isNewFile, false, requestPath) + if !quotaResult.HasSpace { + c.Log(logger.LevelInfo, "denying file write due to quota limits") + return nil, common.ErrQuotaExceeded + } + err := common.ExecutePreAction(&c.User, common.OperationPreUpload, resolvedPath, requestPath, c.GetProtocol(), fileSize, os.O_TRUNC) + if err != nil { + c.Log(logger.LevelDebug, "upload for file %#v denied by pre action: %v", requestPath, err) + return nil, c.GetPermissionDeniedError() + } + + maxWriteSize, _ := c.GetMaxWriteSize(quotaResult, false, fileSize, fs.IsUploadResumeSupported()) + + file, w, cancelFn, err := fs.Create(filePath, 0) + if err != nil { + c.Log(logger.LevelWarn, "error opening existing file, source: %#v, err: %+v", filePath, err) + return nil, c.GetFsError(fs, err) + } + + initialSize := int64(0) + if !isNewFile { + if vfs.IsLocalOrSFTPFs(fs) { + vfolder, err := c.User.GetVirtualFolderForPath(path.Dir(requestPath)) + if err == nil { + dataprovider.UpdateVirtualFolderQuota(&vfolder.BaseVirtualFolder, 0, -fileSize, false) //nolint:errcheck + if vfolder.IsIncludedInUserQuota() { + dataprovider.UpdateUserQuota(&c.User, 0, -fileSize, false) //nolint:errcheck + } + } else { + dataprovider.UpdateUserQuota(&c.User, 0, -fileSize, false) //nolint:errcheck + } + } else { + initialSize = fileSize + } + if maxWriteSize > 0 { + maxWriteSize += fileSize + } + } + + vfs.SetPathPermissions(fs, filePath, c.User.GetUID(), c.User.GetGID()) + + baseTransfer := common.NewBaseTransfer(file, c.BaseConnection, cancelFn, resolvedPath, filePath, requestPath, + common.TransferUpload, 0, initialSize, maxWriteSize, isNewFile, fs) + return newHTTPDFile(baseTransfer, w, nil), nil } diff --git a/httpd/httpd.go b/httpd/httpd.go index 3e44ad56..f77e990b 100644 --- a/httpd/httpd.go +++ b/httpd/httpd.go @@ -56,8 +56,8 @@ const ( adminPwdCompatPath = "/api/v2/changepwd/admin" userPwdPath = "/api/v2/user/changepwd" userPublicKeysPath = "/api/v2/user/publickeys" - userReadFolderPath = "/api/v2/user/folder" - userGetFilePath = "/api/v2/user/file" + userFolderPath = "/api/v2/user/folder" + userFilePath = "/api/v2/user/file" userStreamZipPath = "/api/v2/user/streamzip" healthzPath = "/healthz" webRootPathDefault = "/" @@ -98,6 +98,7 @@ const ( MaxRestoreSize = 10485760 // 10 MB maxRequestSize = 1048576 // 1MB maxLoginPostSize = 262144 // 256 KB + maxMultipartMem = 8388608 // 8MB osWindows = "windows" ) @@ -142,6 +143,8 @@ var ( webChangeClientKeysPath string webClientLogoutPath string webStaticFilesPath string + // max upload size for http clients, 1GB by default + maxUploadFileSize = int64(1048576000) ) func init() { @@ -250,6 +253,9 @@ type Conf struct { // If empty a random signing key will be generated each time SFTPGo starts. If you set a // signing passphrase you should consider rotating it periodically for added security SigningPassphrase string `json:"signing_passphrase" mapstructure:"signing_passphrase"` + // MaxUploadFileSize Defines the maximum request body size, in bytes, for Web Client/API HTTP upload requests. + // 0 means no limit + MaxUploadFileSize int64 `json:"max_upload_file_size" mapstructure:"max_upload_file_size"` } type apiResponse struct { @@ -361,6 +367,7 @@ func (c *Conf) Initialize(configDir string) error { }(binding) } + maxUploadFileSize = c.MaxUploadFileSize startJWTTokensCleanupTicker(tokenDuration) return <-exitChannel } diff --git a/httpd/httpd_test.go b/httpd/httpd_test.go index 66423886..aec25c79 100644 --- a/httpd/httpd_test.go +++ b/httpd/httpd_test.go @@ -78,8 +78,8 @@ const ( logoutPath = "/api/v2/logout" userPwdPath = "/api/v2/user/changepwd" userPublicKeysPath = "/api/v2/user/publickeys" - userReadFolderPath = "/api/v2/user/folder" - userGetFilePath = "/api/v2/user/file" + userFolderPath = "/api/v2/user/folder" + userFilePath = "/api/v2/user/file" userStreamZipPath = "/api/v2/user/streamzip" healthzPath = "/healthz" webBasePath = "/web" @@ -156,7 +156,7 @@ var ( testServer *httptest.Server providerDriverName string postConnectPath string - preDownloadPath string + preActionPath string ) type fakeConnection struct { @@ -212,7 +212,7 @@ func TestMain(m *testing.M) { } postConnectPath = filepath.Join(homeBasePath, "postconnect.sh") - preDownloadPath = filepath.Join(homeBasePath, "predownload.sh") + preActionPath = filepath.Join(homeBasePath, "preaction.sh") httpConfig := config.GetHTTPConfig() httpConfig.Initialize(configDir) //nolint:errcheck @@ -302,7 +302,7 @@ func TestMain(m *testing.M) { os.Remove(hostKeyPath) os.Remove(hostKeyPath + ".pub") os.Remove(postConnectPath) - os.Remove(preDownloadPath) + os.Remove(preActionPath) os.Exit(exitCode) } @@ -4852,14 +4852,14 @@ func TestWebAPILoginMock(t *testing.T) { webToken, err := getJWTWebClientTokenFromTestServer(defaultUsername, defaultPassword) assert.NoError(t, err) // a web token is not valid for API usage - req, err := http.NewRequest(http.MethodGet, userReadFolderPath, nil) + req, err := http.NewRequest(http.MethodGet, userFolderPath, nil) assert.NoError(t, err) setBearerForReq(req, webToken) rr := executeRequest(req) checkResponseCode(t, http.StatusUnauthorized, rr) assert.Contains(t, rr.Body.String(), "Your token audience is not valid") - req, err = http.NewRequest(http.MethodGet, userReadFolderPath+"/?path=%2F", nil) + req, err = http.NewRequest(http.MethodGet, userFolderPath+"/?path=%2F", nil) assert.NoError(t, err) setBearerForReq(req, apiToken) rr = executeRequest(req) @@ -4959,13 +4959,13 @@ func TestWebClientLoginMock(t *testing.T) { checkResponseCode(t, http.StatusNotFound, rr) assert.Contains(t, rr.Body.String(), "Unable to retrieve your user") - req, _ = http.NewRequest(http.MethodGet, userReadFolderPath, nil) + req, _ = http.NewRequest(http.MethodGet, userFolderPath, nil) setBearerForReq(req, apiUserToken) rr = executeRequest(req) checkResponseCode(t, http.StatusNotFound, rr) assert.Contains(t, rr.Body.String(), "Unable to retrieve your user") - req, _ = http.NewRequest(http.MethodGet, userGetFilePath, nil) + req, _ = http.NewRequest(http.MethodGet, userFilePath, nil) setBearerForReq(req, apiUserToken) rr = executeRequest(req) checkResponseCode(t, http.StatusNotFound, rr) @@ -5412,12 +5412,12 @@ func TestPreDownloadHook(t *testing.T) { oldHook := common.Config.Actions.Hook common.Config.Actions.ExecuteOn = []string{common.OperationPreDownload} - common.Config.Actions.Hook = preDownloadPath + common.Config.Actions.Hook = preActionPath u := getTestUser() user, _, err := httpdtest.AddUser(u, http.StatusCreated) assert.NoError(t, err) - err = os.WriteFile(preDownloadPath, getExitCodeScriptContent(0), os.ModePerm) + err = os.WriteFile(preActionPath, getExitCodeScriptContent(0), os.ModePerm) assert.NoError(t, err) testFileName := "testfile" @@ -5438,14 +5438,14 @@ func TestPreDownloadHook(t *testing.T) { checkResponseCode(t, http.StatusOK, rr) assert.Equal(t, testFileContents, rr.Body.Bytes()) - req, err = http.NewRequest(http.MethodGet, userGetFilePath+"?path="+testFileName, nil) + req, err = http.NewRequest(http.MethodGet, userFilePath+"?path="+testFileName, nil) assert.NoError(t, err) setBearerForReq(req, webAPIToken) rr = executeRequest(req) checkResponseCode(t, http.StatusOK, rr) assert.Equal(t, testFileContents, rr.Body.Bytes()) - err = os.WriteFile(preDownloadPath, getExitCodeScriptContent(1), os.ModePerm) + err = os.WriteFile(preActionPath, getExitCodeScriptContent(1), os.ModePerm) assert.NoError(t, err) req, err = http.NewRequest(http.MethodGet, webClientFilesPath+"?path="+testFileName, nil) assert.NoError(t, err) @@ -5454,7 +5454,7 @@ func TestPreDownloadHook(t *testing.T) { checkResponseCode(t, http.StatusOK, rr) assert.Contains(t, rr.Body.String(), "permission denied") - req, err = http.NewRequest(http.MethodGet, userGetFilePath+"?path="+testFileName, nil) + req, err = http.NewRequest(http.MethodGet, userFilePath+"?path="+testFileName, nil) assert.NoError(t, err) setBearerForReq(req, webAPIToken) rr = executeRequest(req) @@ -5470,6 +5470,63 @@ func TestPreDownloadHook(t *testing.T) { common.Config.Actions.Hook = oldHook } +func TestPreUploadHook(t *testing.T) { + if runtime.GOOS == osWindows { + t.Skip("this test is not available on Windows") + } + oldExecuteOn := common.Config.Actions.ExecuteOn + oldHook := common.Config.Actions.Hook + + common.Config.Actions.ExecuteOn = []string{common.OperationPreUpload} + common.Config.Actions.Hook = preActionPath + + u := getTestUser() + user, _, err := httpdtest.AddUser(u, http.StatusCreated) + assert.NoError(t, err) + err = os.WriteFile(preActionPath, getExitCodeScriptContent(0), os.ModePerm) + assert.NoError(t, err) + + webAPIToken, err := getJWTAPIUserTokenFromTestServer(defaultUsername, defaultPassword) + assert.NoError(t, err) + + body := new(bytes.Buffer) + writer := multipart.NewWriter(body) + part, err := writer.CreateFormFile("filename", "filepre") + assert.NoError(t, err) + _, err = part.Write([]byte("file content")) + assert.NoError(t, err) + err = writer.Close() + assert.NoError(t, err) + reader := bytes.NewReader(body.Bytes()) + _, err = reader.Seek(0, io.SeekStart) + assert.NoError(t, err) + req, err := http.NewRequest(http.MethodPost, userFilePath, reader) + assert.NoError(t, err) + req.Header.Add("Content-Type", writer.FormDataContentType()) + setBearerForReq(req, webAPIToken) + rr := executeRequest(req) + checkResponseCode(t, http.StatusCreated, rr) + + err = os.WriteFile(preActionPath, getExitCodeScriptContent(1), os.ModePerm) + assert.NoError(t, err) + _, err = reader.Seek(0, io.SeekStart) + assert.NoError(t, err) + req, err = http.NewRequest(http.MethodPost, userFilePath, reader) + assert.NoError(t, err) + req.Header.Add("Content-Type", writer.FormDataContentType()) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusForbidden, rr) + + _, err = httpdtest.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) + + common.Config.Actions.ExecuteOn = oldExecuteOn + common.Config.Actions.Hook = oldHook +} + func TestWebGetFiles(t *testing.T) { user, _, err := httpdtest.AddUser(getTestUser(), http.StatusCreated) assert.NoError(t, err) @@ -5508,7 +5565,7 @@ func TestWebGetFiles(t *testing.T) { assert.NoError(t, err) assert.Len(t, dirContents, 1) - req, _ = http.NewRequest(http.MethodGet, userReadFolderPath+"?path="+testDir, nil) + req, _ = http.NewRequest(http.MethodGet, userFolderPath+"?path="+testDir, nil) setBearerForReq(req, webAPIToken) rr = executeRequest(req) checkResponseCode(t, http.StatusOK, rr) @@ -5558,7 +5615,7 @@ func TestWebGetFiles(t *testing.T) { assert.NoError(t, err) assert.Len(t, dirContents, len(extensions)+1) - req, _ = http.NewRequest(http.MethodGet, userReadFolderPath+"?path=/", nil) + req, _ = http.NewRequest(http.MethodGet, userFolderPath+"?path=/", nil) setBearerForReq(req, webAPIToken) rr = executeRequest(req) checkResponseCode(t, http.StatusOK, rr) @@ -5573,7 +5630,7 @@ func TestWebGetFiles(t *testing.T) { checkResponseCode(t, http.StatusNotFound, rr) assert.Contains(t, rr.Body.String(), "Unable to get directory contents") - req, _ = http.NewRequest(http.MethodGet, userReadFolderPath+"?path=missing", nil) + req, _ = http.NewRequest(http.MethodGet, userFolderPath+"?path=missing", nil) setBearerForReq(req, webAPIToken) rr = executeRequest(req) checkResponseCode(t, http.StatusNotFound, rr) @@ -5585,25 +5642,25 @@ func TestWebGetFiles(t *testing.T) { checkResponseCode(t, http.StatusOK, rr) assert.Equal(t, testFileContents, rr.Body.Bytes()) - req, _ = http.NewRequest(http.MethodGet, userGetFilePath+"?path="+testFileName, nil) + req, _ = http.NewRequest(http.MethodGet, userFilePath+"?path="+testFileName, nil) setBearerForReq(req, webAPIToken) rr = executeRequest(req) checkResponseCode(t, http.StatusOK, rr) assert.Equal(t, testFileContents, rr.Body.Bytes()) - req, _ = http.NewRequest(http.MethodGet, userGetFilePath+"?path=", nil) + req, _ = http.NewRequest(http.MethodGet, userFilePath+"?path=", nil) setBearerForReq(req, webAPIToken) rr = executeRequest(req) checkResponseCode(t, http.StatusBadRequest, rr) assert.Contains(t, rr.Body.String(), "Please set the path to a valid file") - req, _ = http.NewRequest(http.MethodGet, userGetFilePath+"?path="+testDir, nil) + req, _ = http.NewRequest(http.MethodGet, userFilePath+"?path="+testDir, nil) setBearerForReq(req, webAPIToken) rr = executeRequest(req) checkResponseCode(t, http.StatusBadRequest, rr) assert.Contains(t, rr.Body.String(), "is a directory") - req, _ = http.NewRequest(http.MethodGet, userGetFilePath+"?path=notafile", nil) + req, _ = http.NewRequest(http.MethodGet, userFilePath+"?path=notafile", nil) setBearerForReq(req, webAPIToken) rr = executeRequest(req) checkResponseCode(t, http.StatusNotFound, rr) @@ -5616,7 +5673,7 @@ func TestWebGetFiles(t *testing.T) { checkResponseCode(t, http.StatusPartialContent, rr) assert.Equal(t, testFileContents[2:], rr.Body.Bytes()) - req, _ = http.NewRequest(http.MethodGet, userGetFilePath+"?path="+testFileName, nil) + req, _ = http.NewRequest(http.MethodGet, userFilePath+"?path="+testFileName, nil) req.Header.Set("Range", "bytes=2-") setBearerForReq(req, webAPIToken) rr = executeRequest(req) @@ -5642,7 +5699,7 @@ func TestWebGetFiles(t *testing.T) { rr = executeRequest(req) checkResponseCode(t, http.StatusRequestedRangeNotSatisfiable, rr) - req, _ = http.NewRequest(http.MethodGet, userGetFilePath+"?path="+testFileName, nil) + req, _ = http.NewRequest(http.MethodGet, userFilePath+"?path="+testFileName, nil) req.Header.Set("Range", "bytes=2b-") setBearerForReq(req, webAPIToken) rr = executeRequest(req) @@ -5680,7 +5737,7 @@ func TestWebGetFiles(t *testing.T) { rr = executeRequest(req) checkResponseCode(t, http.StatusPreconditionFailed, rr) - req, _ = http.NewRequest(http.MethodHead, userGetFilePath+"?path="+testFileName, nil) + req, _ = http.NewRequest(http.MethodHead, userFilePath+"?path="+testFileName, nil) req.Header.Set("If-Unmodified-Since", time.Now().UTC().Add(-120*time.Second).Format(http.TimeFormat)) setBearerForReq(req, webAPIToken) rr = executeRequest(req) @@ -5706,12 +5763,12 @@ func TestWebGetFiles(t *testing.T) { rr = executeRequest(req) checkResponseCode(t, http.StatusForbidden, rr) - req, _ = http.NewRequest(http.MethodGet, userGetFilePath+"?path="+testFileName, nil) + req, _ = http.NewRequest(http.MethodGet, userFilePath+"?path="+testFileName, nil) setBearerForReq(req, webAPIToken) rr = executeRequest(req) checkResponseCode(t, http.StatusForbidden, rr) - req, _ = http.NewRequest(http.MethodGet, userReadFolderPath+"?path="+testDir, nil) + req, _ = http.NewRequest(http.MethodGet, userFolderPath+"?path="+testDir, nil) setBearerForReq(req, webAPIToken) rr = executeRequest(req) checkResponseCode(t, http.StatusForbidden, rr) @@ -5739,7 +5796,7 @@ func TestWebGetFiles(t *testing.T) { rr = executeRequest(req) checkResponseCode(t, http.StatusForbidden, rr) - req, _ = http.NewRequest(http.MethodGet, userReadFolderPath+"?path="+testDir, nil) + req, _ = http.NewRequest(http.MethodGet, userFolderPath+"?path="+testDir, nil) setBearerForReq(req, webAPIToken) rr = executeRequest(req) checkResponseCode(t, http.StatusForbidden, rr) @@ -5750,6 +5807,672 @@ func TestWebGetFiles(t *testing.T) { assert.NoError(t, err) } +func TestWebDirsAPI(t *testing.T) { + user, _, err := httpdtest.AddUser(getTestUser(), http.StatusCreated) + assert.NoError(t, err) + webAPIToken, err := getJWTAPIUserTokenFromTestServer(defaultUsername, defaultPassword) + assert.NoError(t, err) + testDir := "testdir" + + req, err := http.NewRequest(http.MethodGet, userFolderPath, nil) + assert.NoError(t, err) + setBearerForReq(req, webAPIToken) + rr := executeRequest(req) + checkResponseCode(t, http.StatusOK, rr) + var contents []map[string]interface{} + err = json.NewDecoder(rr.Body).Decode(&contents) + assert.NoError(t, err) + assert.Len(t, contents, 0) + + // rename a missing folder + req, err = http.NewRequest(http.MethodPatch, userFolderPath+"?path="+testDir+"&target="+testDir+"new", nil) + assert.NoError(t, err) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusNotFound, rr) + // delete a missing folder + req, err = http.NewRequest(http.MethodDelete, userFolderPath+"?path="+testDir, nil) + assert.NoError(t, err) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusNotFound, rr) + // create a dir + req, err = http.NewRequest(http.MethodPost, userFolderPath+"?path="+testDir, nil) + assert.NoError(t, err) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusCreated, rr) + // check the dir was created + req, err = http.NewRequest(http.MethodGet, userFolderPath, nil) + assert.NoError(t, err) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusOK, rr) + contents = nil + err = json.NewDecoder(rr.Body).Decode(&contents) + assert.NoError(t, err) + if assert.Len(t, contents, 1) { + assert.Equal(t, testDir, contents[0]["name"]) + } + // rename the dir + req, err = http.NewRequest(http.MethodPatch, userFolderPath+"?path="+testDir+"&target="+testDir+"new", nil) + assert.NoError(t, err) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusOK, rr) + // delete the dir + req, err = http.NewRequest(http.MethodDelete, userFolderPath+"?path="+testDir+"new", nil) + assert.NoError(t, err) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusOK, rr) + // the root dir cannot be created + req, err = http.NewRequest(http.MethodPost, userFolderPath, nil) + assert.NoError(t, err) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusInternalServerError, rr) + + user.Permissions["/"] = []string{dataprovider.PermListItems} + user, _, err = httpdtest.UpdateUser(user, http.StatusOK, "") + assert.NoError(t, err) + // the user has no more the permission to create the directory + req, err = http.NewRequest(http.MethodPost, userFolderPath+"?path="+testDir, nil) + assert.NoError(t, err) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusForbidden, rr) + + _, err = httpdtest.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) + + // the user is deleted, any API call should fail + req, err = http.NewRequest(http.MethodPost, userFolderPath+"?path="+testDir, nil) + assert.NoError(t, err) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusNotFound, rr) + + req, err = http.NewRequest(http.MethodPatch, userFolderPath+"?path="+testDir+"&target="+testDir+"new", nil) + assert.NoError(t, err) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusNotFound, rr) + + req, err = http.NewRequest(http.MethodDelete, userFolderPath+"?path="+testDir+"new", nil) + assert.NoError(t, err) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusNotFound, rr) +} + +func TestWebFilesAPI(t *testing.T) { + user, _, err := httpdtest.AddUser(getTestUser(), http.StatusCreated) + assert.NoError(t, err) + webAPIToken, err := getJWTAPIUserTokenFromTestServer(defaultUsername, defaultPassword) + assert.NoError(t, err) + + body := new(bytes.Buffer) + writer := multipart.NewWriter(body) + part1, err := writer.CreateFormFile("filename", "file1.txt") + assert.NoError(t, err) + _, err = part1.Write([]byte("file1 content")) + assert.NoError(t, err) + part2, err := writer.CreateFormFile("filename", "file2.txt") + assert.NoError(t, err) + _, err = part2.Write([]byte("file2 content")) + assert.NoError(t, err) + err = writer.Close() + assert.NoError(t, err) + reader := bytes.NewReader(body.Bytes()) + + req, err := http.NewRequest(http.MethodPost, userFilePath, reader) + assert.NoError(t, err) + setBearerForReq(req, webAPIToken) + rr := executeRequest(req) + checkResponseCode(t, http.StatusBadRequest, rr) + assert.Contains(t, rr.Body.String(), "Unable to parse multipart form") + _, err = reader.Seek(0, io.SeekStart) + assert.NoError(t, err) + // set the proper content type + req, err = http.NewRequest(http.MethodPost, userFilePath, reader) + assert.NoError(t, err) + req.Header.Add("Content-Type", writer.FormDataContentType()) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusCreated, rr) + // check we have 2 files + req, err = http.NewRequest(http.MethodGet, userFolderPath, nil) + assert.NoError(t, err) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusOK, rr) + var contents []map[string]interface{} + err = json.NewDecoder(rr.Body).Decode(&contents) + assert.NoError(t, err) + assert.Len(t, contents, 2) + // overwrite the existing files + _, err = reader.Seek(0, io.SeekStart) + assert.NoError(t, err) + req, err = http.NewRequest(http.MethodPost, userFilePath, reader) + assert.NoError(t, err) + req.Header.Add("Content-Type", writer.FormDataContentType()) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusCreated, rr) + req, err = http.NewRequest(http.MethodGet, userFolderPath, nil) + assert.NoError(t, err) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusOK, rr) + contents = nil + err = json.NewDecoder(rr.Body).Decode(&contents) + assert.NoError(t, err) + assert.Len(t, contents, 2) + // now create a dir and upload to that dir + testDir := "tdir" + req, err = http.NewRequest(http.MethodPost, userFolderPath+"?path="+testDir, nil) + assert.NoError(t, err) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusCreated, rr) + _, err = reader.Seek(0, io.SeekStart) + assert.NoError(t, err) + req, err = http.NewRequest(http.MethodPost, userFilePath+"?path="+testDir, reader) + assert.NoError(t, err) + req.Header.Add("Content-Type", writer.FormDataContentType()) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusCreated, rr) + req, err = http.NewRequest(http.MethodGet, userFolderPath, nil) + assert.NoError(t, err) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusOK, rr) + contents = nil + err = json.NewDecoder(rr.Body).Decode(&contents) + assert.NoError(t, err) + assert.Len(t, contents, 3) + req, err = http.NewRequest(http.MethodGet, userFolderPath+"?path="+testDir, nil) + assert.NoError(t, err) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusOK, rr) + contents = nil + err = json.NewDecoder(rr.Body).Decode(&contents) + assert.NoError(t, err) + assert.Len(t, contents, 2) + // rename a file + req, err = http.NewRequest(http.MethodPatch, userFilePath+"?path=file1.txt&target=%2Ftdir%2Ffile3.txt", nil) + assert.NoError(t, err) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusOK, rr) + // rename a missing file + req, err = http.NewRequest(http.MethodPatch, userFilePath+"?path=file1.txt&target=%2Ftdir%2Ffile3.txt", nil) + assert.NoError(t, err) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusNotFound, rr) + // delete a file + req, err = http.NewRequest(http.MethodDelete, userFilePath+"?path=file2.txt", nil) + assert.NoError(t, err) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusOK, rr) + // delete a missing file + req, err = http.NewRequest(http.MethodDelete, userFilePath+"?path=file2.txt", nil) + assert.NoError(t, err) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusNotFound, rr) + // delete a directory + req, err = http.NewRequest(http.MethodDelete, userFilePath+"?path=tdir", nil) + assert.NoError(t, err) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusBadRequest, rr) + // make a symlink outside the home dir and then try to delete it + extPath := filepath.Join(os.TempDir(), "file") + err = os.WriteFile(extPath, []byte("contents"), os.ModePerm) + assert.NoError(t, err) + err = os.Symlink(extPath, filepath.Join(user.GetHomeDir(), "file")) + assert.NoError(t, err) + req, err = http.NewRequest(http.MethodDelete, userFilePath+"?path=file", nil) + assert.NoError(t, err) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusForbidden, rr) + err = os.Remove(extPath) + assert.NoError(t, err) + // remove delete and overwrite permissions + user.Permissions["/"] = []string{dataprovider.PermListItems, dataprovider.PermDownload, dataprovider.PermUpload} + user, _, err = httpdtest.UpdateUser(user, http.StatusOK, "") + assert.NoError(t, err) + _, err = reader.Seek(0, io.SeekStart) + assert.NoError(t, err) + req, err = http.NewRequest(http.MethodPost, userFilePath+"?path=tdir", reader) + assert.NoError(t, err) + req.Header.Add("Content-Type", writer.FormDataContentType()) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusForbidden, rr) + + req, err = http.NewRequest(http.MethodDelete, userFilePath+"?path=%2Ftdir%2Ffile1.txt", nil) + assert.NoError(t, err) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusForbidden, rr) + + _, err = httpdtest.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) + // the user is deleted, any API call should fail + _, err = reader.Seek(0, io.SeekStart) + assert.NoError(t, err) + req, err = http.NewRequest(http.MethodPost, userFilePath, reader) + assert.NoError(t, err) + req.Header.Add("Content-Type", writer.FormDataContentType()) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusNotFound, rr) + + req, err = http.NewRequest(http.MethodPatch, userFilePath+"?path=file1.txt&target=%2Ftdir%2Ffile3.txt", nil) + assert.NoError(t, err) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusNotFound, rr) + + req, err = http.NewRequest(http.MethodDelete, userFilePath+"?path=file2.txt", nil) + assert.NoError(t, err) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusNotFound, rr) +} + +func TestWebUploadErrors(t *testing.T) { + u := getTestUser() + u.QuotaSize = 65535 + subDir1 := "sub1" + subDir2 := "sub2" + u.Permissions[path.Join("/", subDir1)] = []string{dataprovider.PermListItems} + u.Permissions[path.Join("/", subDir2)] = []string{dataprovider.PermListItems, dataprovider.PermUpload, + dataprovider.PermDelete} + u.Filters.FilePatterns = []sdk.PatternsFilter{ + { + Path: "/sub2", + AllowedPatterns: []string{}, + DeniedPatterns: []string{"*.zip"}, + }, + } + user, _, err := httpdtest.AddUser(u, http.StatusCreated) + assert.NoError(t, err) + webAPIToken, err := getJWTAPIUserTokenFromTestServer(defaultUsername, defaultPassword) + assert.NoError(t, err) + + body := new(bytes.Buffer) + writer := multipart.NewWriter(body) + part, err := writer.CreateFormFile("filename", "file.zip") + assert.NoError(t, err) + _, err = part.Write([]byte("file content")) + assert.NoError(t, err) + err = writer.Close() + assert.NoError(t, err) + reader := bytes.NewReader(body.Bytes()) + _, err = reader.Seek(0, io.SeekStart) + assert.NoError(t, err) + // zip file are not allowed within sub2 + req, err := http.NewRequest(http.MethodPost, userFilePath+"?path=sub2", reader) + assert.NoError(t, err) + req.Header.Add("Content-Type", writer.FormDataContentType()) + setBearerForReq(req, webAPIToken) + rr := executeRequest(req) + checkResponseCode(t, http.StatusForbidden, rr) + + _, err = reader.Seek(0, io.SeekStart) + assert.NoError(t, err) + // we have no upload permissions within sub1 + req, err = http.NewRequest(http.MethodPost, userFilePath+"?path=sub1", reader) + assert.NoError(t, err) + req.Header.Add("Content-Type", writer.FormDataContentType()) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusForbidden, rr) + + // create a dir and try to overwrite it with a file + req, err = http.NewRequest(http.MethodPost, userFolderPath+"?path=file.zip", nil) + assert.NoError(t, err) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusCreated, rr) + + _, err = reader.Seek(0, io.SeekStart) + assert.NoError(t, err) + req, err = http.NewRequest(http.MethodPost, userFilePath, reader) + assert.NoError(t, err) + req.Header.Add("Content-Type", writer.FormDataContentType()) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusInternalServerError, rr) + assert.Contains(t, rr.Body.String(), "operation unsupported") + // try to upload to a missing parent directory + _, err = reader.Seek(0, io.SeekStart) + assert.NoError(t, err) + req, err = http.NewRequest(http.MethodPost, userFilePath+"?path=missingdir", reader) + assert.NoError(t, err) + req.Header.Add("Content-Type", writer.FormDataContentType()) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusNotFound, rr) + + req, err = http.NewRequest(http.MethodDelete, userFolderPath+"?path=file.zip", nil) + assert.NoError(t, err) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusOK, rr) + // upload will work now + _, err = reader.Seek(0, io.SeekStart) + assert.NoError(t, err) + req, err = http.NewRequest(http.MethodPost, userFilePath, reader) + assert.NoError(t, err) + req.Header.Add("Content-Type", writer.FormDataContentType()) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusCreated, rr) + // overwrite the file + _, err = reader.Seek(0, io.SeekStart) + assert.NoError(t, err) + req, err = http.NewRequest(http.MethodPost, userFilePath, reader) + assert.NoError(t, err) + req.Header.Add("Content-Type", writer.FormDataContentType()) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusCreated, rr) + + vfs.SetTempPath(filepath.Join(os.TempDir(), "missingpath")) + + _, err = reader.Seek(0, io.SeekStart) + assert.NoError(t, err) + req, err = http.NewRequest(http.MethodPost, userFilePath, reader) + assert.NoError(t, err) + req.Header.Add("Content-Type", writer.FormDataContentType()) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusNotFound, rr) + + if runtime.GOOS != osWindows { + req, err = http.NewRequest(http.MethodDelete, userFilePath+"?path=file.zip", reader) + assert.NoError(t, err) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusOK, rr) + + vfs.SetTempPath(filepath.Clean(os.TempDir())) + err = os.Chmod(user.GetHomeDir(), 0555) + assert.NoError(t, err) + + _, err = reader.Seek(0, io.SeekStart) + assert.NoError(t, err) + req, err = http.NewRequest(http.MethodPost, userFilePath, reader) + assert.NoError(t, err) + req.Header.Add("Content-Type", writer.FormDataContentType()) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusForbidden, rr) + assert.Contains(t, rr.Body.String(), "Error closing file") + + err = os.Chmod(user.GetHomeDir(), os.ModePerm) + assert.NoError(t, err) + } + + vfs.SetTempPath("") + + // upload a multipart form with no files + body = new(bytes.Buffer) + writer = multipart.NewWriter(body) + err = writer.Close() + assert.NoError(t, err) + reader = bytes.NewReader(body.Bytes()) + _, err = reader.Seek(0, io.SeekStart) + assert.NoError(t, err) + req, err = http.NewRequest(http.MethodPost, userFilePath+"?path=sub2", reader) + assert.NoError(t, err) + req.Header.Add("Content-Type", writer.FormDataContentType()) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusBadRequest, rr) + assert.Contains(t, rr.Body.String(), "No files uploaded!") + + _, err = httpdtest.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) +} + +func TestWebAPIVFolder(t *testing.T) { + u := getTestUser() + u.QuotaSize = 65535 + vdir := "/vdir" + mappedPath := filepath.Join(os.TempDir(), "vdir") + folderName := filepath.Base(mappedPath) + u.VirtualFolders = append(u.VirtualFolders, vfs.VirtualFolder{ + BaseVirtualFolder: vfs.BaseVirtualFolder{ + Name: folderName, + MappedPath: mappedPath, + }, + VirtualPath: vdir, + QuotaSize: -1, + QuotaFiles: -1, + }) + user, _, err := httpdtest.AddUser(u, http.StatusCreated) + assert.NoError(t, err) + + webAPIToken, err := getJWTAPIUserTokenFromTestServer(user.Username, defaultPassword) + assert.NoError(t, err) + + fileContents := []byte("test contents") + + body := new(bytes.Buffer) + writer := multipart.NewWriter(body) + part, err := writer.CreateFormFile("filename", "file.txt") + assert.NoError(t, err) + _, err = part.Write(fileContents) + assert.NoError(t, err) + err = writer.Close() + assert.NoError(t, err) + reader := bytes.NewReader(body.Bytes()) + + req, err := http.NewRequest(http.MethodPost, userFilePath+"?path=vdir", reader) + assert.NoError(t, err) + req.Header.Add("Content-Type", writer.FormDataContentType()) + setBearerForReq(req, webAPIToken) + rr := executeRequest(req) + checkResponseCode(t, http.StatusCreated, rr) + + user, _, err = httpdtest.GetUserByUsername(user.Username, http.StatusOK) + assert.NoError(t, err) + assert.Equal(t, int64(len(fileContents)), user.UsedQuotaSize) + + folder, _, err := httpdtest.GetFolderByName(folderName, http.StatusOK) + assert.NoError(t, err) + assert.Equal(t, int64(len(fileContents)), folder.UsedQuotaSize) + + _, err = reader.Seek(0, io.SeekStart) + assert.NoError(t, err) + req, err = http.NewRequest(http.MethodPost, userFilePath+"?path=vdir", reader) + assert.NoError(t, err) + req.Header.Add("Content-Type", writer.FormDataContentType()) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusCreated, rr) + + user, _, err = httpdtest.GetUserByUsername(user.Username, http.StatusOK) + assert.NoError(t, err) + assert.Equal(t, int64(len(fileContents)), user.UsedQuotaSize) + + folder, _, err = httpdtest.GetFolderByName(folderName, http.StatusOK) + assert.NoError(t, err) + assert.Equal(t, int64(len(fileContents)), folder.UsedQuotaSize) + + _, err = httpdtest.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) + _, err = httpdtest.RemoveFolder(vfs.BaseVirtualFolder{Name: folderName}, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) + err = os.RemoveAll(mappedPath) + assert.NoError(t, err) +} + +func TestWebAPICryptFs(t *testing.T) { + u := getTestUser() + u.QuotaSize = 65535 + u.FsConfig.Provider = sdk.CryptedFilesystemProvider + u.FsConfig.CryptConfig.Passphrase = kms.NewPlainSecret(defaultPassword) + user, _, err := httpdtest.AddUser(u, http.StatusCreated) + assert.NoError(t, err) + webAPIToken, err := getJWTAPIUserTokenFromTestServer(defaultUsername, defaultPassword) + assert.NoError(t, err) + + body := new(bytes.Buffer) + writer := multipart.NewWriter(body) + part, err := writer.CreateFormFile("filename", "file.txt") + assert.NoError(t, err) + _, err = part.Write([]byte("content")) + assert.NoError(t, err) + err = writer.Close() + assert.NoError(t, err) + reader := bytes.NewReader(body.Bytes()) + + req, err := http.NewRequest(http.MethodPost, userFilePath, reader) + assert.NoError(t, err) + req.Header.Add("Content-Type", writer.FormDataContentType()) + setBearerForReq(req, webAPIToken) + rr := executeRequest(req) + checkResponseCode(t, http.StatusCreated, rr) + + _, err = reader.Seek(0, io.SeekStart) + assert.NoError(t, err) + req, err = http.NewRequest(http.MethodPost, userFilePath, reader) + assert.NoError(t, err) + req.Header.Add("Content-Type", writer.FormDataContentType()) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusCreated, rr) + + _, err = httpdtest.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) +} + +func TestWebUploadSFTP(t *testing.T) { + u := getTestUser() + localUser, _, err := httpdtest.AddUser(u, http.StatusCreated) + assert.NoError(t, err) + u = getTestSFTPUser() + u.QuotaFiles = 100 + u.FsConfig.SFTPConfig.BufferSize = 2 + u.HomeDir = filepath.Join(os.TempDir(), u.Username) + sftpUser, _, err := httpdtest.AddUser(u, http.StatusCreated) + assert.NoError(t, err) + + webAPIToken, err := getJWTAPIUserTokenFromTestServer(sftpUser.Username, defaultPassword) + assert.NoError(t, err) + + body := new(bytes.Buffer) + writer := multipart.NewWriter(body) + part, err := writer.CreateFormFile("filename", "file.txt") + assert.NoError(t, err) + _, err = part.Write([]byte("test file content")) + assert.NoError(t, err) + err = writer.Close() + assert.NoError(t, err) + reader := bytes.NewReader(body.Bytes()) + + req, err := http.NewRequest(http.MethodPost, userFilePath, reader) + assert.NoError(t, err) + req.Header.Add("Content-Type", writer.FormDataContentType()) + setBearerForReq(req, webAPIToken) + rr := executeRequest(req) + checkResponseCode(t, http.StatusCreated, rr) + + expectedQuotaSize := int64(17) + expectedQuotaFiles := 1 + user, _, err := httpdtest.GetUserByUsername(sftpUser.Username, http.StatusOK) + assert.NoError(t, err) + assert.Equal(t, expectedQuotaFiles, user.UsedQuotaFiles) + assert.Equal(t, expectedQuotaSize, user.UsedQuotaSize) + + user.QuotaSize = 10 + user, _, err = httpdtest.UpdateUser(user, http.StatusOK, "") + assert.NoError(t, err) + // we are now overquota on overwrite + _, err = reader.Seek(0, io.SeekStart) + assert.NoError(t, err) + req, err = http.NewRequest(http.MethodPost, userFilePath, reader) + assert.NoError(t, err) + req.Header.Add("Content-Type", writer.FormDataContentType()) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusInternalServerError, rr) + assert.Contains(t, rr.Body.String(), "denying write due to space limit") + // delete the file + req, err = http.NewRequest(http.MethodDelete, userFilePath+"?path=file.txt", nil) + assert.NoError(t, err) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusOK, rr) + + _, err = reader.Seek(0, io.SeekStart) + assert.NoError(t, err) + req, err = http.NewRequest(http.MethodPost, userFilePath, reader) + assert.NoError(t, err) + req.Header.Add("Content-Type", writer.FormDataContentType()) + setBearerForReq(req, webAPIToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusInternalServerError, rr) + assert.Contains(t, rr.Body.String(), "denying write due to space limit") + + _, err = httpdtest.RemoveUser(sftpUser, http.StatusOK) + assert.NoError(t, err) + _, err = httpdtest.RemoveUser(localUser, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(localUser.GetHomeDir()) + assert.NoError(t, err) + err = os.RemoveAll(sftpUser.GetHomeDir()) + assert.NoError(t, err) +} + +func TestWebUploadMultipartFormReadError(t *testing.T) { + user, _, err := httpdtest.AddUser(getTestUser(), http.StatusCreated) + assert.NoError(t, err) + webAPIToken, err := getJWTAPIUserTokenFromTestServer(defaultUsername, defaultPassword) + assert.NoError(t, err) + + req, err := http.NewRequest(http.MethodPost, userFilePath, nil) + assert.NoError(t, err) + + mpartForm := &multipart.Form{ + File: make(map[string][]*multipart.FileHeader), + } + mpartForm.File["filename"] = append(mpartForm.File["filename"], &multipart.FileHeader{Filename: "missing"}) + req.MultipartForm = mpartForm + req.Header.Add("Content-Type", "multipart/form-data") + setBearerForReq(req, webAPIToken) + rr := executeRequest(req) + checkResponseCode(t, http.StatusInternalServerError, rr) + assert.Contains(t, rr.Body.String(), "Unable to read uploaded file") + + _, err = httpdtest.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) +} + func TestCompressionErrorMock(t *testing.T) { user, _, err := httpdtest.AddUser(getTestUser(), http.StatusCreated) assert.NoError(t, err) @@ -5872,21 +6595,49 @@ func TestClientUserClose(t *testing.T) { testFilePath := filepath.Join(user.GetHomeDir(), testFileName) err = createTestFile(testFilePath, testFileSize) assert.NoError(t, err) + uploadContent := make([]byte, testFileSize) + _, err = rand.Read(uploadContent) + assert.NoError(t, err) webToken, err := getJWTWebClientTokenFromTestServer(defaultUsername, defaultPassword) assert.NoError(t, err) + webAPIToken, err := getJWTAPIUserTokenFromTestServer(defaultUsername, defaultPassword) + assert.NoError(t, err) + var wg sync.WaitGroup wg.Add(1) go func() { + defer wg.Done() req, _ := http.NewRequest(http.MethodGet, webClientFilesPath+"?path="+testFileName, nil) setJWTCookieForReq(req, webToken) rr := executeRequest(req) checkResponseCode(t, http.StatusOK, rr) - wg.Done() }() - + wg.Add(1) + go func() { + defer wg.Done() + body := new(bytes.Buffer) + writer := multipart.NewWriter(body) + part, err := writer.CreateFormFile("filename", "upload.dat") + assert.NoError(t, err) + n, err := part.Write(uploadContent) + assert.NoError(t, err) + assert.Equal(t, testFileSize, int64(n)) + err = writer.Close() + assert.NoError(t, err) + reader := bytes.NewReader(body.Bytes()) + req, err := http.NewRequest(http.MethodPost, userFilePath, reader) + assert.NoError(t, err) + req.Header.Add("Content-Type", writer.FormDataContentType()) + setBearerForReq(req, webAPIToken) + rr := executeRequest(req) + checkResponseCode(t, http.StatusInternalServerError, rr) + assert.Contains(t, rr.Body.String(), "transfer aborted") + }() + // wait for the transfers assert.Eventually(t, func() bool { - for _, stat := range common.Connections.GetStats() { - if len(stat.Transfers) > 0 { + stats := common.Connections.GetStats() + if len(stats) == 2 { + if len(stats[0].Transfers) > 0 && len(stats[1].Transfers) > 0 { return true } } @@ -5894,6 +6645,7 @@ func TestClientUserClose(t *testing.T) { }, 1*time.Second, 50*time.Millisecond) for _, stat := range common.Connections.GetStats() { + // close all the active transfers common.Connections.Close(stat.ConnectionID) } wg.Wait() diff --git a/httpd/internal_test.go b/httpd/internal_test.go index f73c4524..181fabaf 100644 --- a/httpd/internal_test.go +++ b/httpd/internal_test.go @@ -1456,6 +1456,39 @@ func TestConnection(t *testing.T) { assert.ErrorIs(t, err, os.ErrNotExist) } +func TestGetFileWriterErrors(t *testing.T) { + user := dataprovider.User{ + BaseUser: sdk.BaseUser{ + Username: "test_httpd_user", + HomeDir: "invalid", + }, + } + user.Permissions = make(map[string][]string) + user.Permissions["/"] = []string{dataprovider.PermAny} + connection := &Connection{ + BaseConnection: common.NewBaseConnection(xid.New().String(), common.ProtocolHTTP, "", user), + request: nil, + } + _, err := connection.getFileWriter("name") + assert.Error(t, err) + + user.FsConfig.Provider = sdk.S3FilesystemProvider + user.FsConfig.S3Config = vfs.S3FsConfig{ + S3FsConfig: sdk.S3FsConfig{ + Bucket: "b", + Region: "us-west-1", + AccessKey: "key", + AccessSecret: kms.NewPlainSecret("secret"), + }, + } + connection = &Connection{ + BaseConnection: common.NewBaseConnection(xid.New().String(), common.ProtocolHTTP, "", user), + request: nil, + } + _, err = connection.getFileWriter("/path") + assert.Error(t, err) +} + func TestHTTPDFile(t *testing.T) { user := dataprovider.User{ BaseUser: sdk.BaseUser{ @@ -1484,7 +1517,7 @@ func TestHTTPDFile(t *testing.T) { baseTransfer := common.NewBaseTransfer(file, connection.BaseConnection, nil, p, p, name, common.TransferDownload, 0, 0, 0, false, fs) - httpdFile := newHTTPDFile(baseTransfer, nil) + httpdFile := newHTTPDFile(baseTransfer, nil, nil) // the file is closed, read should fail buf := make([]byte, 100) _, err = httpdFile.Read(buf) @@ -1495,6 +1528,14 @@ func TestHTTPDFile(t *testing.T) { assert.ErrorIs(t, err, common.ErrTransferClosed) err = os.Remove(p) assert.NoError(t, err) + + httpdFile.writer = file + httpdFile.File = nil + httpdFile.ErrTransfer = nil + err = httpdFile.closeIO() + assert.Error(t, err) + assert.Error(t, httpdFile.ErrTransfer) + assert.Equal(t, err, httpdFile.ErrTransfer) } func TestChangeUserPwd(t *testing.T) { diff --git a/httpd/schema/openapi.yaml b/httpd/schema/openapi.yaml index 3a8216a6..27d52b0d 100644 --- a/httpd/schema/openapi.yaml +++ b/httpd/schema/openapi.yaml @@ -17,7 +17,7 @@ info: Several storage backends are supported and they are configurable per user, so you can serve a local directory for a user and an S3 bucket (or part of it) for another one. SFTPGo also supports virtual folders, a virtual folder can use any of the supported storage backends. So you can have, for example, an S3 user that exposes a GCS bucket (or part of it) on a specified path and an encrypted local filesystem on another one. Virtual folders can be private or shared among multiple users, for shared virtual folders you can define different quota limits for each user. - version: 2.1.0 + version: 2.1.0-dev contact: name: API support url: 'https://github.com/drakkan/sftpgo' @@ -1790,6 +1790,108 @@ paths: $ref: '#/components/responses/InternalServerError' default: $ref: '#/components/responses/DefaultResponse' + post: + tags: + - users API + summary: Create a directory + description: Create a directory for the logged in user + operationId: create_user_folder + parameters: + - in: query + name: path + description: Path to the folder to create. It must be URL encoded, for example the path "my dir/àdir" must be sent as "my%20dir%2F%C3%A0dir" + schema: + type: string + required: true + responses: + '201': + description: successful operation + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/ApiResponse' + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + '500': + $ref: '#/components/responses/InternalServerError' + default: + $ref: '#/components/responses/DefaultResponse' + patch: + tags: + - users API + summary: Rename a directory + description: Rename a directory for the logged in user. The rename is allowed for empty directory or for non empty, local directories, with no virtual folders inside + operationId: rename_user_folder + parameters: + - in: query + name: path + description: Path to the folder to rename. It must be URL encoded, for example the path "my dir/àdir" must be sent as "my%20dir%2F%C3%A0dir" + schema: + type: string + required: true + - in: query + name: target + description: New name. It must be URL encoded, for example the path "my dir/àdir" must be sent as "my%20dir%2F%C3%A0dir" + schema: + type: string + required: true + responses: + '200': + description: successful operation + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/ApiResponse' + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + '500': + $ref: '#/components/responses/InternalServerError' + default: + $ref: '#/components/responses/DefaultResponse' + delete: + tags: + - users API + summary: Delete a directory + description: Delete a directory for the logged in user. Only empty directories can be deleted + operationId: delete_user_folder + parameters: + - in: query + name: path + description: Path to the folder to delete. It must be URL encoded, for example the path "my dir/àdir" must be sent as "my%20dir%2F%C3%A0dir" + schema: + type: string + required: true + responses: + '200': + description: successful operation + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/ApiResponse' + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + '500': + $ref: '#/components/responses/InternalServerError' + default: + $ref: '#/components/responses/DefaultResponse' /user/file: get: tags: @@ -1829,6 +1931,121 @@ paths: $ref: '#/components/responses/InternalServerError' default: $ref: '#/components/responses/DefaultResponse' + post: + tags: + - users API + summary: Upload files + description: Upload one or more files for the logged in user + operationId: create_user_files + parameters: + - in: query + name: path + description: Parent directory for the uploaded files. It must be URL encoded, for example the path "my dir/àdir" must be sent as "my%20dir%2F%C3%A0dir". If empty or missing the root path is assumed. If a file with the same name already exists, it will be overwritten + schema: + type: string + requestBody: + content: + multipart/form-data: + schema: + type: object + properties: + filename: + type: array + items: + type: string + format: binary + minItems: 1 + uniqueItems: true + required: true + responses: + '201': + description: successful operation + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/ApiResponse' + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + '500': + $ref: '#/components/responses/InternalServerError' + default: + $ref: '#/components/responses/DefaultResponse' + patch: + tags: + - users API + summary: Rename afile + description: Rename a file for the logged in user + operationId: rename_user_file + parameters: + - in: query + name: path + description: Path to the file to rename. It must be URL encoded + schema: + type: string + required: true + - in: query + name: target + description: New name. It must be URL encoded + schema: + type: string + required: true + responses: + '200': + description: successful operation + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/ApiResponse' + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + '500': + $ref: '#/components/responses/InternalServerError' + default: + $ref: '#/components/responses/DefaultResponse' + delete: + tags: + - users API + summary: Delete a file + description: Delete a file for the logged in user. + operationId: delete_user_file + parameters: + - in: query + name: path + description: Path to the file to delete. It must be URL encoded + schema: + type: string + required: true + responses: + '200': + description: successful operation + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/ApiResponse' + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + '500': + $ref: '#/components/responses/InternalServerError' + default: + $ref: '#/components/responses/DefaultResponse' /user/streamzip: post: tags: diff --git a/httpd/server.go b/httpd/server.go index 6f9fbe86..9807a10a 100644 --- a/httpd/server.go +++ b/httpd/server.go @@ -631,8 +631,14 @@ func (s *httpdServer) initializeRouter() { router.Put(userPwdPath, changeUserPassword) router.With(checkHTTPUserPerm(sdk.WebClientPubKeyChangeDisabled)).Get(userPublicKeysPath, getUserPublicKeys) router.With(checkHTTPUserPerm(sdk.WebClientPubKeyChangeDisabled)).Put(userPublicKeysPath, setUserPublicKeys) - router.Get(userReadFolderPath, readUserFolder) - router.Get(userGetFilePath, getUserFile) + router.Get(userFolderPath, readUserFolder) + router.Post(userFolderPath, createUserDir) + router.Patch(userFolderPath, renameUserDir) + router.Delete(userFolderPath, deleteUserDir) + router.Get(userFilePath, getUserFile) + router.Post(userFilePath, uploadUserFiles) + router.Patch(userFilePath, renameUserFile) + router.Delete(userFilePath, deleteUserFile) router.Post(userStreamZipPath, getUserFilesAsZipStream) }) diff --git a/sftpgo.json b/sftpgo.json index 1ab36c07..8d6e86db 100644 --- a/sftpgo.json +++ b/sftpgo.json @@ -202,7 +202,8 @@ "certificate_key_file": "", "ca_certificates": [], "ca_revocation_lists": [], - "signing_passphrase": "" + "signing_passphrase": "", + "max_upload_file_size": 1048576000 }, "telemetry": { "bind_port": 10000,